const std = @import("std");
const builtin = @import("builtin");
pub const cli = @import("zit").cli;
const zit = @import("zit");
const artifact = zit.artifact;
const checkout = zit.checkout;
const delta = zit.delta;
const fast_export = zit.git_fast_export;
const git_import = zit.git_import;
const logstore = zit.logstore;
const pack = zit.pack;
const storage = zit.storage;
const editor = zit.submanifest_editor;
const verifier = @import("git_import_verify.zig");
pub const ChildLifecycle = struct {
launched: usize = 0,
terminated: usize = 0,
stderr_drained: usize = 0,
reaped: usize = 0,
};
/// What a destination's map records, for an import that adds to it.
pub const KnownHistory = struct {
/// Every ref's peeled commit, each once. The preflight walk stops here.
tips: []const []const u8,
/// Every commit the destination holds. The export excludes all of them,
/// not only those beneath a tip: a recorded tip the source no longer
/// holds (a force-pushed branch, a deleted one, in a fresh fetch) would
/// otherwise leave its ancestors in the export.
commits: []const []const u8,
};
/// The child-facing half of the import transaction. The caller owns the
/// implementation and its backing buffers for the full call to `run`.
pub const ChildFactory = struct {
context: *anyopaque,
vtable: *const VTable,
pub const VTable = struct {
/// `known` is null for a fresh import. For an import that adds to a
/// destination it is what the destination's map records, whose
/// commits the export excludes (`exportArgvInto`).
start_export: *const fn (context: *anyopaque, source: []const u8, known: ?KnownHistory) anyerror!ExportPipes,
/// What identifies the source across fetches: its object format and
/// its parentless commits. An import that adds to a destination
/// compares these with what the destination's map recorded, so a
/// map is never applied to a different history. Null when the
/// factory cannot answer, which refuses such an import.
describe_source: ?*const fn (context: *anyopaque, gpa: std.mem.Allocator, source: []const u8) anyerror!SourceIdentity = null,
/// True/false when the captured first-parent path proves the answer;
/// null when a shallow cut prevents a decision.
first_parent_contains: ?*const fn (context: *anyopaque, source: []const u8, ancestor: []const u8, descendant: []const u8) anyerror!?bool = null,
terminate: *const fn (context: *anyopaque) anyerror!void,
drain_stderr: *const fn (context: *anyopaque) anyerror!void,
reap: *const fn (context: *anyopaque) anyerror!void,
startup_diagnostics: ?*const fn (context: *anyopaque, err: *std.Io.Writer) void = null,
};
};
/// The facts that identify a source history. Caller-owned; `deinit` frees.
pub const SourceIdentity = struct {
/// `sha1` or `sha256`, as `git rev-parse --show-object-format` prints.
object_format: []u8,
/// Every commit with no parent, lowercase hex, sorted ascending.
roots: [][]u8,
/// Every ref the source publishes now, with the commit it resolves to
/// (a tag peeled to its commit). An import that adds to a destination
/// reconciles names against this rather than against the export stream,
/// which omits every ref whose commit the destination already holds.
refs: []SourceRef,
/// The commits a shallow source's history is cut off at, lowercase hex,
/// sorted ascending; empty for a complete source. Git reports each one as
/// parentless, so none of them is in `roots`.
shallow: [][]u8 = &.{},
pub fn deinit(self: *SourceIdentity, gpa: std.mem.Allocator) void {
gpa.free(self.object_format);
for (self.roots) |root| gpa.free(root);
gpa.free(self.roots);
for (self.shallow) |commit| gpa.free(commit);
gpa.free(self.shallow);
for (self.refs) |ref| {
gpa.free(ref.name);
gpa.free(ref.oid);
}
gpa.free(self.refs);
self.* = undefined;
}
};
pub const SourceRef = struct {
name: []u8,
oid: []u8,
};
pub const ExportPipes = struct {
stdout: std.Io.Reader,
stderr: std.Io.Reader,
/// Scripted sessions supply their terminal status here. The OS binding gets
/// its status during reap; an import succeeds only after the stream reaches
/// `done` and EOF, so a nonzero scripted result is still an import failure.
exit_code: u8 = 0,
/// OS-backed sessions publish the wait status here during `reap`; scripted
/// sessions use `exit_code` directly.
exit_code_ref: ?*const u8 = null,
owner: ?*ProductionGit = null,
diagnostics: ?*const fn (pipes: *ExportPipes, err: *std.Io.Writer, enabled: bool) void = null,
detected_default_branch: ?[]const u8 = null,
/// draft-fossil-submanifests-00 §6.3's optimum for the source's own name
/// set, measured during preflight. Null when the source could not be
/// listed or cannot partition; the caller then keeps its own default.
analyzed_submanifest_k: ?u4 = null,
/// The source's commit count, measured during preflight so the ingest
/// loop can report progress against a real denominator. Null when the
/// count could not be taken; progress then reports without a total.
commit_total: ?u64 = null,
/// Squash merges paired with the pull request each one took, read from
/// the source's `refs/pull/N/head` (`SquashPicks`). Null without any.
squash_picks: ?*const SquashPicks = null,
};
/// A squash merge on the default branch and the pull request head it took,
/// by Git OID, so the import can record the squash as a cherry-pick (`Q`)
/// of that head. GitHub keeps `refs/pull/N/head` for every pull request,
/// after its branch is deleted too, and its squash commit's subject ends in
/// ` (#N)`; together they name the pair exactly rather than guess it.
pub const SquashPicks = struct {
/// Squash commit OID to pull-request head OID, both lowercase hex.
by_commit: std.StringHashMapUnmanaged([]const u8) = .empty,
arena: std.heap.ArenaAllocator,
pub fn init(gpa: std.mem.Allocator) SquashPicks {
return .{ .arena = std.heap.ArenaAllocator.init(gpa) };
}
pub fn deinit(self: *SquashPicks) void {
self.arena.deinit();
self.* = undefined;
}
pub fn count(self: *const SquashPicks) usize {
return self.by_commit.count();
}
};
/// The most squash merges paired in one import.
const max_squash_picks: usize = 1 << 20;
/// The pull request a squash commit's subject names: the N of a trailing
/// ` (#N)`, GitHub's default squash subject. Null for any other subject.
pub fn squashPullNumber(subject: []const u8) ?u32 {
const trimmed = std.mem.trimEnd(u8, subject, " \t\r");
if (trimmed.len < 5 or trimmed[trimmed.len - 1] != ')') return null;
const open = std.mem.lastIndexOf(u8, trimmed, " (#") orelse return null;
const digits = trimmed[open + 3 .. trimmed.len - 1];
if (digits.len == 0 or digits.len > 9) return null;
for (digits) |c| if (!std.ascii.isDigit(c)) return null;
return std.fmt.parseInt(u32, digits, 10) catch null;
}
/// The N of `refs/pull/N/head`, or null for any other ref.
pub fn pullHeadNumber(ref: []const u8) ?u32 {
const prefix = "refs/pull/";
const suffix = "/head";
if (!std.mem.startsWith(u8, ref, prefix) or !std.mem.endsWith(u8, ref, suffix)) return null;
if (ref.len <= prefix.len + suffix.len) return null;
const digits = ref[prefix.len .. ref.len - suffix.len];
if (digits.len > 9) return null;
for (digits) |c| if (!std.ascii.isDigit(c)) return null;
return std.fmt.parseInt(u32, digits, 10) catch null;
}
/// The store's §7.2 default policy for a fresh import, derived from the
/// algorithm the import actually names artifacts under rather than from a
/// guess about what it produced. Both call sites read it, so the header and
/// the persisted setting cannot drift apart.
fn importDefaultPolicy() artifact.HashPolicy {
const algo = git_import.import_hash_algo;
return artifact.defaultPolicy(algo == .sha1, algo == .sha3_256);
}
/// The `git fast-export` command line the importer runs, as a value.
///
/// Pure and public so the one policy decision it encodes — what happens to a
/// signature Fossil has no card for — is testable without spawning a process.
/// `refuse_signed` moves exactly the two signature modes and nothing else.
///
/// `--reencode=yes` converts a commit that declares a non-UTF-8 encoding
/// (`encoding ISO-8859-1`) to UTF-8, as Fossil comments are; without it Git
/// stops at the first such commit. A message with no encoding header is
/// passed through as its bytes.
pub fn exportArgv(source: []const u8, refuse_signed: bool) [11][]const u8 {
return .{
"git",
"-C",
source,
"fast-export",
"--all",
"--show-original-ids",
"--mark-tags",
if (refuse_signed) "--signed-commits=abort" else "--signed-commits=strip",
if (refuse_signed) "--signed-tags=abort" else "--signed-tags=strip",
"--reencode=yes",
"--use-done-feature",
};
}
/// The same command line for an import that adds to a destination. Every
/// commit the destination holds is excluded, read from stdin as `^OID` lines
/// (a large history overflows an argument list), and a new commit whose
/// parent is excluded names it by OID (`from <oid>`) rather than losing it.
/// Without `--reference-excluded-parents` Git drops the parent line and the
/// first new commit arrives as a root, which an import would store as a
/// second disconnected history with no error.
///
/// A tag on an excluded commit would otherwise abort the export (Git's
/// default for a tag of an unexported object); it is dropped from the stream
/// instead, and the runner reconciles tags against the map separately.
///
/// A known commit the source does not hold (an abandoned tip, history below
/// a shallow cut) is skipped rather than aborting the export.
pub fn exportArgvInto(source: []const u8, refuse_signed: bool) [15][]const u8 {
return exportArgv(source, refuse_signed) ++ [_][]const u8{ "--reference-excluded-parents", "--tag-of-filtered-object=drop", "--ignore-missing", "--stdin" };
}
/// The `--stdin` input that excludes `commits`. Caller owns the bytes.
pub fn exclusionInput(gpa: std.mem.Allocator, commits: []const []const u8) ![]u8 {
var bytes: std.ArrayList(u8) = .empty;
errdefer bytes.deinit(gpa);
for (commits) |commit| {
try bytes.append(gpa, '^');
try bytes.appendSlice(gpa, commit);
try bytes.append(gpa, '\n');
}
return bytes.toOwnedSlice(gpa);
}
/// OS-backed production dependency. It owns both pipe buffers through the
/// import call; the `ExportPipes` readers retain a pointer to this value.
pub const ProductionGit = struct {
pub const StartupFailure = enum { after_spawn, after_stderr_handoff, after_stderr_thread };
/// Keep room in fixed CLI writers for the authoritative import diagnostic.
const cli_diagnostic_reserve_bytes = 256;
/// Bytes of output one preflight check may produce. Every check asks a
/// bounded question — one ref, one boolean, one config value, or (for
/// `fsck --connectivity-only --no-dangling` and `rev-list --quiet`)
/// nothing at all when the repository is intact — so an overrun means a
/// check's shape was misjudged, never that the source is merely large.
const preflight_output_limit_bytes = 8 * 1024;
gpa: std.mem.Allocator,
io: std.Io,
/// The process environment the spawned git children inherit (plus the
/// GIT_* pins `childEnvironment` adds). Threaded from main's `Init` in
/// production and `std.testing.environ` in tests.
environ: std.process.Environ,
child: ?std.process.Child = null,
exit_code: u8 = 1,
stdout_buffer: [8192]u8 = undefined,
stderr_buffer: [8192]u8 = undefined,
/// Room for an export command line with `--date-order` appended.
argv_buf: [24][]const u8 = undefined,
reaped: bool = false,
reap_calls: usize = 0,
/// Captured by the stdout reader before any parser-error cleanup can
/// terminate the child. This distinguishes an independently completed
/// truncated export from cancellation closing the pipe.
stdout_reached_eof: bool = false,
default_branch: ?[]const u8 = null,
owns_default_branch: bool = false,
analyzed_submanifest_k: ?u4 = null,
commit_total: ?u64 = null,
squash_picks: ?SquashPicks = null,
/// The operator's explicit opt-out from the default signature policy.
/// False (the default) strips; true restores the refusal.
refuse_signed: bool = false,
stderr_thread: ?std.Thread = null,
/// Moved out of `child` before its reader thread starts, so draining never
/// races cleanup or a platform-specific process handle implementation.
stderr_file: ?std.Io.File = null,
stderr_ring: [4096]u8 = undefined,
stderr_ring_start: usize = 0,
stderr_ring_len: usize = 0,
stderr_bytes_drained: usize = 0,
stderr_lock: std.Io.Mutex = .init,
test_argv: ?[]const []const u8 = null,
test_startup_failure: ?StartupFailure = null,
captured_source: ?[]u8 = null,
pub fn init(gpa: std.mem.Allocator, io: std.Io, environ: std.process.Environ) ProductionGit {
return .{ .gpa = gpa, .io = io, .environ = environ };
}
/// Test-only process constructor: uses the identical spawn, stream,
/// concurrent-diagnostic, termination, and reap path as `git fast-export`.
pub fn initForTest(gpa: std.mem.Allocator, io: std.Io, environ: std.process.Environ, argv: []const []const u8) ProductionGit {
return .{ .gpa = gpa, .io = io, .environ = environ, .test_argv = argv };
}
fn childEnvironment(self: *ProductionGit) !std.process.Environ.Map {
// The process environ, threaded from main's `Init` — never `std.c`:
// libc is implicit on macOS but must be explicitly linked on Linux,
// and the environment does not need libc to read.
var env = try self.environ.createMap(self.gpa);
errdefer env.deinit();
try env.put("GIT_TERMINAL_PROMPT", "0");
try env.put("GIT_NO_LAZY_FETCH", "1");
try env.put("GIT_NO_REPLACE_OBJECTS", "1");
try env.put("GIT_CONFIG_NOSYSTEM", "1");
try env.put("GIT_CONFIG_GLOBAL", if (builtin.os.tag == .windows) "NUL" else "/dev/null");
try env.put("GIT_CONFIG_COUNT", "0");
for ([_][]const u8{ "GIT_DIR", "GIT_COMMON_DIR", "GIT_WORK_TREE", "GIT_INDEX_FILE", "GIT_CONFIG_PARAMETERS", "GIT_OBJECT_DIRECTORY", "GIT_ALTERNATE_OBJECT_DIRECTORIES", "GIT_SHALLOW_FILE", "GIT_DEFAULT_REF_FORMAT", "GIT_TEMPLATE_DIR" }) |key| _ = env.swapRemove(key);
return env;
}
pub fn setDefaultBranch(self: *ProductionGit, default_branch: ?[]const u8) void {
if (self.owns_default_branch) self.gpa.free(self.default_branch.?);
self.default_branch = default_branch;
self.owns_default_branch = false;
}
/// Set before `run`, alongside `setDefaultBranch`: the export argv is
/// built inside `start_export`, which the factory vtable does not let the
/// caller parameterize.
pub fn setRefuseSigned(self: *ProductionGit, refuse: bool) void {
self.refuse_signed = refuse;
}
pub fn deinit(self: *ProductionGit) void {
if (self.owns_default_branch) self.gpa.free(self.default_branch.?);
if (self.squash_picks) |*picks| picks.deinit();
if (self.captured_source) |path| {
std.Io.Dir.cwd().deleteTree(self.io, path) catch {};
self.gpa.free(path);
}
self.* = undefined;
}
/// Freeze refs and shallow cuts in disposable private Git metadata while
/// borrowing immutable objects. All proof and export reads use this view;
/// moving the caller's refs later cannot change the selected history.
fn captureSource(self: *ProductionGit, source: []const u8) ![]const u8 {
if (self.captured_source) |path| return path;
const objects_out = try self.query("rev-parse object directory", &.{ "git", "-C", source, "--no-lazy-fetch", "rev-parse", "--path-format=absolute", "--git-path", "objects" });
defer self.gpa.free(objects_out);
const objects = std.mem.trim(u8, objects_out, " \r\n");
if (!std.fs.path.isAbsolute(objects) or std.mem.indexOfAny(u8, objects, "\r\n") != null) return error.InvalidObjectDirectory;
const format_out = try self.query("object format", &.{ "git", "-C", source, "--no-lazy-fetch", "rev-parse", "--show-object-format" });
defer self.gpa.free(format_out);
if (!std.mem.eql(u8, std.mem.trim(u8, format_out, " \r\n"), "sha1")) return error.UnsupportedObjectFormat;
const partial = try self.optionalQuery("partial source", &.{ "git", "-C", source, "--no-lazy-fetch", "config", "--get-regexp", "^(extensions\\.partialclone|remote\\..*\\.promisor)$" });
if (partial) |value| {
defer self.gpa.free(value);
if (std.mem.indexOf(u8, value, "extensions.partialclone ") != null or try containsGitTrue(value)) return error.PartialRepository;
}
const refs = try self.queryCollection("capture refs", &.{ "git", "-C", source, "--no-lazy-fetch", "for-each-ref", "--format=%(objectname) %(refname)" });
defer self.gpa.free(refs);
if (std.mem.indexOf(u8, refs, " refs/replace/") != null) return error.ReplacementRefsUnsupported;
const cuts = try self.shallowBoundary(self.gpa, source);
defer {
for (cuts) |cut| self.gpa.free(cut);
self.gpa.free(cuts);
}
const symbolic = try self.optionalQuery("capture HEAD", &.{ "git", "-C", source, "--no-lazy-fetch", "symbolic-ref", "--quiet", "HEAD" });
defer if (symbolic) |text| self.gpa.free(text);
const head = if (symbolic == null) try self.query("capture detached HEAD", &.{ "git", "-C", source, "--no-lazy-fetch", "rev-parse", "--verify", "HEAD" }) else null;
defer if (head) |text| self.gpa.free(text);
var env = try self.childEnvironment();
defer env.deinit();
const temporary = env.get("TMPDIR") orelse if (builtin.os.tag == .windows) env.get("TEMP") orelse return error.NoTemporaryDirectory else "/tmp";
var nonce: [16]u8 = undefined;
self.io.random(&nonce);
const hex = std.fmt.bytesToHex(nonce, .lower);
const path = try std.fmt.allocPrint(self.gpa, "{s}/zit-import-source-{s}", .{ temporary, hex });
errdefer self.gpa.free(path);
try std.Io.Dir.cwd().createDir(self.io, path, if (builtin.os.tag == .windows) .default_dir else .fromMode(0o700));
errdefer std.Io.Dir.cwd().deleteTree(self.io, path) catch {};
const init_out = try self.query("initialize captured source", &.{ "git", "-c", "init.templateDir=", "init", "--quiet", "--object-format=sha1", "--ref-format=files", path });
self.gpa.free(init_out);
var captured = try std.Io.Dir.cwd().openDir(self.io, path, .{});
defer captured.close(self.io);
const alternate = try std.mem.concat(self.gpa, u8, &.{ objects, "\n" });
defer self.gpa.free(alternate);
try captured.writeFile(self.io, .{ .sub_path = ".git/objects/info/alternates", .data = alternate });
const packed_refs = try std.mem.concat(self.gpa, u8, &.{ "# pack-refs with: sorted\n", refs });
defer self.gpa.free(packed_refs);
try captured.writeFile(self.io, .{ .sub_path = ".git/packed-refs", .data = packed_refs });
const private_head = if (symbolic) |name| try std.fmt.allocPrint(self.gpa, "ref: {s}\n", .{std.mem.trim(u8, name, " \r\n")}) else try self.gpa.dupe(u8, head.?);
defer self.gpa.free(private_head);
try captured.writeFile(self.io, .{ .sub_path = ".git/HEAD", .data = private_head });
if (cuts.len != 0) {
var cut_bytes: std.ArrayList(u8) = .empty;
defer cut_bytes.deinit(self.gpa);
for (cuts) |cut| {
try cut_bytes.appendSlice(self.gpa, cut);
try cut_bytes.append(self.gpa, '\n');
}
try captured.writeFile(self.io, .{ .sub_path = ".git/shallow", .data = cut_bytes.items });
}
self.captured_source = path;
return path;
}
pub fn dependencies(self: *ProductionGit, publication: PublicationHooks) Dependencies {
return .{
.git = .{ .context = self, .vtable = &vtable },
.publication = publication,
};
}
const vtable: ChildFactory.VTable = .{
.start_export = startExport,
.describe_source = describeSource,
.first_parent_contains = firstParentContains,
.terminate = terminate,
.drain_stderr = drainStderr,
.reap = reap,
.startup_diagnostics = writeStartupDiagnostics,
};
fn firstParentContains(context: *anyopaque, requested_source: []const u8, ancestor: []const u8, descendant: []const u8) !?bool {
const self: *ProductionGit = @ptrCast(@alignCast(context));
if (std.mem.eql(u8, ancestor, descendant)) return true;
const source = try self.captureSource(requested_source);
const cuts = try self.shallowBoundary(self.gpa, source);
defer {
for (cuts) |cut| self.gpa.free(cut);
self.gpa.free(cuts);
}
var cursor = try self.gpa.dupe(u8, descendant);
defer self.gpa.free(cursor);
// Each bounded page advances to the parent of its last member. Stop
// as soon as the ancestor or an undecidable shallow boundary appears.
for (0..std.math.maxInt(u32)) |_| {
const output = try self.query("captured first-parent ancestry", &.{ "git", "-C", source, "--no-lazy-fetch", "rev-list", "--first-parent", "--max-count=128", cursor, "--" });
defer self.gpa.free(output);
var last: ?[]const u8 = null;
var lines = std.mem.tokenizeAny(u8, output, "\r\n");
while (lines.next()) |oid| {
if (std.mem.eql(u8, oid, ancestor)) return true;
for (cuts) |cut| if (std.mem.eql(u8, oid, cut)) return null;
last = oid;
}
const end = last orelse return error.InvalidCapturedHistory;
const pair = try self.query("advance captured ancestry", &.{ "git", "-C", source, "--no-lazy-fetch", "rev-list", "--first-parent", "--max-count=2", end, "--" });
defer self.gpa.free(pair);
var parents = std.mem.tokenizeAny(u8, pair, "\r\n");
if (!std.mem.eql(u8, parents.next() orelse return error.InvalidCapturedHistory, end)) return error.InvalidCapturedHistory;
const next = parents.next() orelse return false;
if (std.mem.eql(u8, next, cursor)) return error.InvalidCapturedHistory;
const owned = try self.gpa.dupe(u8, next);
self.gpa.free(cursor);
cursor = owned;
}
return error.CapturedHistoryTooLarge;
}
fn describeSource(context: *anyopaque, gpa: std.mem.Allocator, requested_source: []const u8) !SourceIdentity {
const self: *ProductionGit = @ptrCast(@alignCast(context));
// A test-constructed process stands in for the export and has no
// repository to ask, just as `startExport` skips its preflight.
if (self.test_argv != null) {
const object_format = try gpa.dupe(u8, "sha1");
errdefer gpa.free(object_format);
const roots = try gpa.alloc([]u8, 0);
errdefer gpa.free(roots);
return .{ .object_format = object_format, .roots = roots, .refs = try gpa.alloc(SourceRef, 0) };
}
const source = try self.captureSource(requested_source);
const format_out = try self.query("rev-parse --show-object-format", &.{ "git", "-C", source, "--no-lazy-fetch", "rev-parse", "--show-object-format" });
defer self.gpa.free(format_out);
const object_format = try gpa.dupe(u8, std.mem.trim(u8, format_out, " \r\n"));
errdefer gpa.free(object_format);
const shallow = try self.shallowBoundary(gpa, source);
errdefer {
for (shallow) |commit| gpa.free(commit);
gpa.free(shallow);
}
const roots_out = try self.queryCollection("rev-list --max-parents=0 --all", &.{ "git", "-C", source, "--no-lazy-fetch", "rev-list", "--max-parents=0", "--all" });
defer self.gpa.free(roots_out);
var roots: std.ArrayList([]u8) = .empty;
errdefer {
for (roots.items) |root| gpa.free(root);
roots.deinit(gpa);
}
var lines = std.mem.tokenizeAny(u8, roots_out, "\r\n");
while (lines.next()) |line| {
if (line.len != 40 and line.len != 64) return error.MalformedRootList;
for (line) |byte| if (!std.ascii.isHex(byte)) return error.MalformedRootList;
const root = try std.ascii.allocLowerString(gpa, line);
// A commit the cut left parentless is not a root of the history.
if (std.sort.binarySearch([]u8, shallow, root, orderOid) != null) {
gpa.free(root);
continue;
}
errdefer gpa.free(root);
try roots.append(gpa, root);
}
std.mem.sort([]u8, roots.items, {}, oidLessThan);
// `%(*objectname)` is the peeled commit of an annotated tag and empty
// for everything else; NUL separators because a ref name may hold
// any byte but NUL and newline.
const refs_out = try self.queryCollection("for-each-ref", &.{ "git", "-C", source, "--no-lazy-fetch", "for-each-ref", "--format=%(refname)%00%(objectname)%00%(*objectname)%00%(objecttype)%00%(*objecttype)" });
defer self.gpa.free(refs_out);
var refs: std.ArrayList(SourceRef) = .empty;
errdefer {
for (refs.items) |ref| {
gpa.free(ref.name);
gpa.free(ref.oid);
}
refs.deinit(gpa);
}
var ref_lines = std.mem.tokenizeAny(u8, refs_out, "\r\n");
while (ref_lines.next()) |line| {
var fields = std.mem.splitScalar(u8, line, 0);
const name = fields.next() orelse return error.MalformedRefList;
const object = fields.next() orelse return error.MalformedRefList;
const peeled = fields.next() orelse return error.MalformedRefList;
const object_type = fields.next() orelse return error.MalformedRefList;
const peeled_type = fields.next() orelse return error.MalformedRefList;
// A ref that resolves to no commit (a tag of a tree or blob)
// names nothing an import can carry, so it is not a ref here.
const commit = if (peeled.len != 0) (if (std.mem.eql(u8, peeled_type, "commit")) peeled else continue) else (if (std.mem.eql(u8, object_type, "commit")) object else continue);
const owned_name = try gpa.dupe(u8, name);
errdefer gpa.free(owned_name);
const owned_oid = try std.ascii.allocLowerString(gpa, commit);
errdefer gpa.free(owned_oid);
try refs.append(gpa, .{ .name = owned_name, .oid = owned_oid });
}
const owned_roots = try roots.toOwnedSlice(gpa);
errdefer {
for (owned_roots) |root| gpa.free(root);
gpa.free(owned_roots);
}
return .{ .object_format = object_format, .roots = owned_roots, .refs = try refs.toOwnedSlice(gpa), .shallow = shallow };
}
fn oidLessThan(_: void, a: []u8, b: []u8) bool {
return std.mem.lessThan(u8, a, b);
}
fn orderOid(key: []u8, item: []u8) std.math.Order {
return std.mem.order(u8, key, item);
}
/// Bytes of `.git/shallow` the source may hold: one line per commit its
/// history is cut off at, so this bounds the cut, not the repository.
const shallow_file_limit_bytes = 16 << 20;
/// The commits a shallow source is cut off at, lowercase hex, sorted;
/// empty when the source is complete. Git keeps them in one file and has
/// no command that lists them, so the file is read where Git says it is.
fn shallowBoundary(self: *ProductionGit, gpa: std.mem.Allocator, source: []const u8) ![][]u8 {
const path_out = try self.query("rev-parse --git-path shallow", &.{ "git", "-C", source, "--no-lazy-fetch", "rev-parse", "--path-format=absolute", "--git-path", "shallow" });
defer self.gpa.free(path_out);
const path = std.mem.trim(u8, path_out, " \t\r\n");
const bytes = std.Io.Dir.cwd().readFileAlloc(self.io, path, self.gpa, .limited(shallow_file_limit_bytes)) catch |err| switch (err) {
error.FileNotFound => return gpa.alloc([]u8, 0),
else => return err,
};
defer self.gpa.free(bytes);
var commits: std.ArrayList([]u8) = .empty;
errdefer {
for (commits.items) |commit| gpa.free(commit);
commits.deinit(gpa);
}
var lines = std.mem.tokenizeAny(u8, bytes, "\r\n");
while (lines.next()) |line| {
if (line.len != 40 and line.len != 64) return error.MalformedShallowFile;
for (line) |byte| if (!std.ascii.isHex(byte)) return error.MalformedShallowFile;
const commit = try std.ascii.allocLowerString(gpa, line);
errdefer gpa.free(commit);
try commits.append(gpa, commit);
}
std.mem.sort([]u8, commits.items, {}, oidLessThan);
return commits.toOwnedSlice(gpa);
}
fn startExport(context: *anyopaque, requested_source: []const u8, known: ?KnownHistory) !ExportPipes {
const self: *ProductionGit = @ptrCast(@alignCast(context));
if (self.test_argv) |argv| return self.startSession(argv, null);
const known_tips: ?[]const []const u8 = if (known) |k| k.tips else null;
const first_capture = self.captured_source == null;
if (first_capture) try self.preflight(requested_source, known_tips);
const source = try self.captureSource(requested_source);
// Auto-selected HEAD must belong to the captured view, not an earlier
// preflight observation of refs the caller can still move.
if (first_capture and self.owns_default_branch) self.setDefaultBranch(null);
try self.preflight(source, known_tips);
try self.findSquashPicks(source);
// Fossil has no signature card, so a signature cannot survive this
// import in any representation. `strip` accepts that loss and is the
// default — refusing by default only withheld the import, never the
// signature. `abort` is the operator's explicit opt-out. There is no
// honest third option: a verbatim pass would have to invent somewhere
// to keep bytes the format cannot hold.
if (known) |k| {
// The exclusions live in the private capture, so the export reads
// them from a file rather than a pipe it could block on. A shallow
// source's cut commits are among them: the runner has refused any
// cut its map does not know.
const input = try exclusionInput(self.gpa, k.commits);
defer self.gpa.free(input);
var captured = try std.Io.Dir.cwd().openDir(self.io, source, .{});
defer captured.close(self.io);
try captured.writeFile(self.io, .{ .sub_path = ".git/zit-known-commits", .data = input });
const stdin = try captured.openFile(self.io, ".git/zit-known-commits", .{});
defer stdin.close(self.io);
const argv = exportArgvInto(source, self.refuse_signed);
return self.startSession(self.ordered(&argv), stdin);
}
const argv = exportArgv(source, self.refuse_signed);
return self.startSession(self.ordered(&argv), null);
}
/// `argv`, plus `--date-order` when squash merges were paired. A squash
/// and the pull request it took share no ancestry, so only commit dates
/// put the head first, and the squash can name a head it has seen. Any
/// other import keeps the default order, and so its exact bytes.
fn ordered(self: *ProductionGit, argv: []const []const u8) []const []const u8 {
const picks = self.squash_picks orelse return argv;
if (picks.count() == 0) return argv;
std.debug.assert(argv.len < self.argv_buf.len);
@memcpy(self.argv_buf[0..argv.len], argv);
self.argv_buf[argv.len] = "--date-order";
return self.argv_buf[0 .. argv.len + 1];
}
/// Pair each squash merge on the default branch's first-parent chain
/// with the pull request head it took. A commit qualifies when it has
/// one parent, its subject ends ` (#N)`, `refs/pull/N/head` exists, and
/// that head is not already its ancestor (a merge or fast-forward,
/// which needs no `Q`). A source without pull refs costs one query.
fn findSquashPicks(self: *ProductionGit, source: []const u8) !void {
const branch = self.default_branch orelse return;
const refs = try self.queryCollection("for-each-ref refs/pull", &.{ "git", "-C", source, "--no-lazy-fetch", "for-each-ref", "--format=%(refname) %(objectname)", "refs/pull" });
defer self.gpa.free(refs);
var picks = SquashPicks.init(self.gpa);
errdefer picks.deinit();
const a = picks.arena.allocator();
var heads: std.AutoHashMapUnmanaged(u32, []const u8) = .empty;
var ref_lines = std.mem.tokenizeScalar(u8, refs, '\n');
while (ref_lines.next()) |line| {
const space = std.mem.indexOfScalar(u8, line, ' ') orelse continue;
const number = pullHeadNumber(line[0..space]) orelse continue;
try heads.put(a, number, try std.ascii.allocLowerString(a, line[space + 1 ..]));
}
if (heads.count() == 0) {
picks.deinit();
return;
}
const branch_ref = try std.fmt.allocPrint(a, "refs/heads/{s}", .{branch});
const log = try self.queryCollection("log --first-parent", &.{ "git", "-C", source, "--no-lazy-fetch", "log", "--first-parent", "--format=%H %P%x1f%s", branch_ref, "--" });
defer self.gpa.free(log);
var commits = std.mem.tokenizeScalar(u8, log, '\n');
while (commits.next()) |line| {
const sep = std.mem.indexOfScalar(u8, line, 0x1f) orelse continue;
var ids = std.mem.tokenizeScalar(u8, line[0..sep], ' ');
const commit = ids.next() orelse continue;
_ = ids.next() orelse continue; // a root takes nothing
if (ids.next() != null) continue; // a merge already joins its branch
const number = squashPullNumber(line[sep + 1 ..]) orelse continue;
const head = heads.get(number) orelse continue;
if (std.ascii.eqlIgnoreCase(head, commit)) continue;
// Exit 0 says the head is an ancestor: nothing to pick.
const ancestor = try self.optionalQuery("merge-base --is-ancestor", &.{ "git", "-C", source, "--no-lazy-fetch", "merge-base", "--is-ancestor", head, commit });
if (ancestor) |bytes| {
self.gpa.free(bytes);
continue;
}
if (picks.count() == max_squash_picks) break;
try picks.by_commit.put(a, try std.ascii.allocLowerString(a, commit), head);
}
if (self.squash_picks) |*old| old.deinit();
self.squash_picks = picks;
}
fn startSession(self: *ProductionGit, argv: []const []const u8, stdin: ?std.Io.File) !ExportPipes {
var env = try self.childEnvironment();
defer env.deinit();
self.reaped = false;
self.exit_code = 1;
self.reap_calls = 0;
self.stdout_reached_eof = false;
self.stderr_file = null;
self.child = try std.process.spawn(self.io, .{
.argv = argv,
.environ_map = &env,
.stdin = if (stdin) |file| .{ .file = file } else .ignore,
.stdout = .pipe,
.stderr = .pipe,
});
errdefer self.cleanupFailedStart();
try self.failStartForTest(.after_spawn);
const child = &(self.child orelse unreachable);
const stderr = child.stderr orelse return error.MissingStderr;
child.stderr = null;
self.stderr_file = stderr;
try self.failStartForTest(.after_stderr_handoff);
self.stderr_ring_start = 0;
self.stderr_bytes_drained = 0;
self.stderr_ring_len = 0;
self.stderr_thread = try std.Thread.spawn(.{}, drainStderrThread, .{self});
try self.failStartForTest(.after_stderr_thread);
return .{
.exit_code_ref = &self.exit_code,
.detected_default_branch = self.default_branch,
.analyzed_submanifest_k = self.analyzed_submanifest_k,
.commit_total = self.commit_total,
.squash_picks = if (self.squash_picks) |*picks| (if (picks.count() != 0) picks else null) else null,
.owner = self,
.stdout = .{
.vtable = &stdout_reader_vtable,
.buffer = &self.stdout_buffer,
.seek = 0,
.end = 0,
},
.stderr = .{
.vtable = &stderr_reader_vtable,
.buffer = &self.stderr_buffer,
.seek = 0,
.end = 0,
},
.diagnostics = writeDiagnostics,
};
}
fn failStartForTest(self: *const ProductionGit, point: StartupFailure) !void {
if (self.test_startup_failure == point) return error.InjectedStartFailure;
}
fn cleanupFailedStart(self: *ProductionGit) void {
if (self.stderr_thread == null) {
if (self.stderr_file) |file| file.close(self.io);
self.stderr_file = null;
}
if (self.child) |*child| {
if (child.stderr) |file| file.close(self.io);
child.stderr = null;
}
terminate(@ptrCast(self)) catch {};
reap(@ptrCast(self)) catch {};
}
/// `known_tips` is the destination's tips for an import that adds to it,
/// null for a fresh import.
fn preflight(self: *ProductionGit, source: []const u8, known_tips: ?[]const []const u8) !void {
// FIRST, and without --no-lazy-fetch: every other preflight query
// carries that flag, and a git older than 2.45 rejects the whole
// command line with a usage dump — 13 opaque e2e failures on stock
// Ubuntu 24.04 read that way before this check named the cause.
// The requirement stays hard rather than version-conditional argv:
// the flag's guarantee is already re-provided by the shallow/partial
// rejections below, and a second code path whose only purpose is
// re-proving that would be complexity with no carrier.
const version_out = try self.query("--version", &.{ "git", "--version" });
defer self.gpa.free(version_out);
const version_line = std.mem.trim(u8, version_out, " \t\r\n");
if (gitOlderThan(version_line, 2, 45)) {
var buf: [160]u8 = undefined;
const msg = std.fmt.bufPrint(
&buf,
"zit import-git needs git >= 2.45 (for --no-lazy-fetch); found: {s}\n",
.{version_line},
) catch "zit import-git needs git >= 2.45 (for --no-lazy-fetch)\n";
self.retainDiagnostics(msg);
return error.GitTooOld;
}
// A bare clone is a source too: the export reads refs and objects,
// never a work tree. git answers one line per flag.
const kind = try self.query("rev-parse --is-bare-repository --is-inside-work-tree", &.{ "git", "-C", source, "--no-lazy-fetch", "rev-parse", "--is-bare-repository", "--is-inside-work-tree" });
defer self.gpa.free(kind);
var answers = std.mem.tokenizeAny(u8, kind, " \t\r\n");
const bare = answers.next() orelse "";
const inside = answers.next() orelse "";
if (!std.mem.eql(u8, bare, "true") and !std.mem.eql(u8, inside, "true")) return error.NotGitRepository;
const head = try self.query("rev-parse HEAD", &.{ "git", "-C", source, "--no-lazy-fetch", "rev-parse", "HEAD" });
defer self.gpa.free(head);
if (std.mem.trim(u8, head, " \t\r\n").len == 0) return error.MissingHead;
const object_format = try self.query("rev-parse --show-object-format", &.{ "git", "-C", source, "--no-lazy-fetch", "rev-parse", "--show-object-format" });
defer self.gpa.free(object_format);
if (!std.mem.eql(u8, std.mem.trim(u8, object_format, " \t\r\n"), "sha1")) return error.UnsupportedObjectFormat;
const shallow = try self.query("rev-parse --is-shallow-repository", &.{ "git", "-C", source, "--no-lazy-fetch", "rev-parse", "--is-shallow-repository" });
defer self.gpa.free(shallow);
// A fresh import has nothing below a cut to stand on, so it would
// write the cut commits as roots. An import that adds to a
// destination may stand on what the destination holds: the runner
// has already refused a cut at any commit its map does not know, so
// what the export names by OID beneath the new commits is in DEST.
if (std.mem.eql(u8, std.mem.trim(u8, shallow, " \t\r\n"), "true") and known_tips == null) return error.ShallowRepository;
if (known_tips) |tips| {
try self.checkNewObjects(source, tips);
} else try self.checkConnectivity(source);
const symbolic = try self.optionalQuery("symbolic-ref HEAD", &.{ "git", "-C", source, "--no-lazy-fetch", "symbolic-ref", "--quiet", "HEAD" });
if (symbolic) |branch_output| {
defer self.gpa.free(branch_output);
if (self.default_branch == null) {
const branch_ref = std.mem.trim(u8, branch_output, " \t\r\n");
const prefix = "refs/heads/";
if (!std.mem.startsWith(u8, branch_ref, prefix) or branch_ref.len == prefix.len) return error.InvalidHead;
self.default_branch = try self.gpa.dupe(u8, branch_ref[prefix.len..]);
self.owns_default_branch = true;
}
} else if (self.default_branch == null) {
return error.DetachedHead;
}
const partial_clone = try self.optionalQuery("config --get extensions.partialClone", &.{ "git", "-C", source, "--no-lazy-fetch", "config", "--get", "extensions.partialClone" });
if (partial_clone) |configured_remote| {
defer self.gpa.free(configured_remote);
if (std.mem.trim(u8, configured_remote, " \t\r\n").len != 0) return error.PartialRepository;
}
const promisors = try self.optionalQuery("config --get-regexp remote.*.promisor", &.{ "git", "-C", source, "--no-lazy-fetch", "config", "--get-regexp", "^remote\\..*\\.promisor$" });
if (promisors) |configured_promisors| {
defer self.gpa.free(configured_promisors);
if (try containsGitTrue(configured_promisors)) return error.PartialRepository;
}
const counted = try self.optionalQuery("rev-list --count --all", &.{ "git", "-C", source, "--no-lazy-fetch", "rev-list", "--count", "--all" });
if (counted) |count_output| {
defer self.gpa.free(count_output);
self.commit_total = std.fmt.parseInt(u64, std.mem.trim(u8, count_output, " \t\r\n"), 10) catch null;
}
self.analyzed_submanifest_k = try self.analyzeSubmanifestPin(source);
}
/// Every object reachable in the source is present: the whole-repository
/// check, for a fresh import, which reads all of it.
fn checkConnectivity(self: *ProductionGit, source: []const u8) !void {
// `--no-dangling` is required, not tidiness. A dangling object is
// unreachable but intact, so it says nothing about the connectivity
// question this check exists to answer, yet git reports one
// informational line per object on stdout: penpot emitted 14,763 bytes
// and a healthy repository failed its import on the output limit.
// A missing object is still reported, and still makes this check exit
// non-zero.
const connectivity = self.query("fsck --connectivity-only", &.{ "git", "-C", source, "--no-lazy-fetch", "fsck", "--connectivity-only", "--no-dangling", "--no-reflogs" }) catch |err| switch (err) {
// The exit status is this check's verdict; its stdout is a
// per-problem report — with `--no-dangling` an intact repository
// writes none of it. So an overrun here is not an unreadable
// answer, it is a repository with more broken links than the
// report can hold. The named-overrun diagnostic is already
// retained, so the operator sees both which check overran and the
// verdict it produced.
error.GitPreflightFailed, error.PreflightOutputTooLarge => return error.MissingObjects,
else => return err,
};
defer self.gpa.free(connectivity);
}
/// Every object the export of new commits will read is present, at the
/// cost of those commits alone. The walk starts from every ref, as the
/// export does, and stops at each known tip, as the export does.
/// `--ignore-missing` drops a known tip the source does not hold — an
/// old tag outside a shallow clone, a deleted branch gc removed — which
/// the walk could not reach anyway; the export drops it the same way. A
/// commit beneath it that the walk does reach is one the destination
/// already holds, and the importer refuses to write a commit twice.
fn checkNewObjects(self: *ProductionGit, source: []const u8, known_tips: []const []const u8) !void {
const fixed = [_][]const u8{ "git", "-C", source, "--no-lazy-fetch", "rev-list", "--objects", "--missing=error", "--quiet", "--ignore-missing", "--all", "--not" };
const argv = try self.gpa.alloc([]const u8, fixed.len + known_tips.len);
defer self.gpa.free(argv);
@memcpy(argv[0..fixed.len], &fixed);
@memcpy(argv[fixed.len..], known_tips);
// `--quiet` leaves the exit status as the whole answer.
const connectivity = self.query("rev-list --objects --missing=error", argv) catch |err| switch (err) {
error.GitPreflightFailed, error.PreflightOutputTooLarge => return error.MissingObjects,
else => return err,
};
self.gpa.free(connectivity);
}
/// `git ls-tree` output the §6.3 analysis is willing to read. A monorepo
/// at the ceiling lists several million paths; past it the pin falls back
/// to the default rather than failing an import over an advisory
/// measurement.
const ls_tree_limit_bytes = 64 << 20;
/// Measure §6.3's optimum for the source before a single object is read.
/// The tip tree, not the root commit: an imported history's first commit
/// is routinely a lone README, and pinning k from it would misconfigure
/// every later commit in the repository.
///
/// Advisory throughout — every failure path returns null and leaves the
/// caller's default standing. `-z --name-only` is required, not stylistic:
/// without it git quotes unusual bytes and the names would no longer be
/// the ones §6.1's partition predicate hashes. The listing includes paths
/// a check-in's F-set would exclude (§4.6.3 reserved names) and any path
/// `update` would reject; both are rare enough not to move an optimum that
/// tracks half the base-2 logarithm of the count.
fn analyzeSubmanifestPin(self: *ProductionGit, source: []const u8) !?u4 {
var env = try self.childEnvironment();
defer env.deinit();
const result = std.process.run(self.gpa, self.io, .{
.argv = &.{ "git", "-C", source, "--no-lazy-fetch", "ls-tree", "-r", "-z", "--name-only", "--full-tree", "HEAD" },
.environ_map = &env,
.stdout_limit = .limited(ls_tree_limit_bytes),
.stderr_limit = .limited(8 * 1024),
}) catch return null;
defer self.gpa.free(result.stderr);
defer self.gpa.free(result.stdout);
switch (result.term) {
.exited => |code| if (code != 0) return null,
else => return null,
}
var names: std.ArrayList([]const u8) = .empty;
defer names.deinit(self.gpa);
var listing = std.mem.splitScalar(u8, result.stdout, 0);
while (listing.next()) |name| {
if (name.len == 0) continue;
try names.append(self.gpa, name);
}
return git_import.analyzeSubmanifestK(self.gpa, names.items, git_import.import_hash_algo);
}
fn containsGitTrue(configured: []const u8) !bool {
var lines = std.mem.splitScalar(u8, configured, '\n');
while (lines.next()) |line| {
const trimmed = std.mem.trim(u8, line, " \t\r");
if (trimmed.len == 0) continue;
const value = trimmed[(std.mem.lastIndexOfAny(u8, trimmed, " \t") orelse return error.InvalidGitBoolean) + 1 ..];
if (try isGitTrue(value)) return true;
}
return false;
}
fn isGitTrue(value: []const u8) !bool {
if (std.mem.eql(u8, value, "1") or std.ascii.eqlIgnoreCase(value, "true") or
std.ascii.eqlIgnoreCase(value, "yes") or std.ascii.eqlIgnoreCase(value, "on")) return true;
if (std.mem.eql(u8, value, "0") or std.ascii.eqlIgnoreCase(value, "false") or
std.ascii.eqlIgnoreCase(value, "no") or std.ascii.eqlIgnoreCase(value, "off")) return false;
return error.InvalidGitBoolean;
}
/// A limit overrun names the check that overran. `StreamTooLong` alone
/// tells an operator nothing actionable: which of the preflight children
/// produced too much output is the entire diagnostic.
fn preflightOverran(self: *ProductionGit, check: []const u8) error{PreflightOutputTooLarge} {
var buffer: [256]u8 = undefined;
const message = std.fmt.bufPrint(
&buffer,
"import-git: preflight check 'git {s}' produced more than {d} bytes of output\n",
.{ check, preflight_output_limit_bytes },
) catch fallback: {
// Every `check` is a short call-site literal, well inside the
// buffer; a truncating format would be a bug here, not a wire
// condition. Still total: report without the name rather than
// dropping the diagnostic.
break :fallback "import-git: a preflight check exceeded its output limit\n";
};
self.retainDiagnostics(message);
return error.PreflightOutputTooLarge;
}
fn optionalQuery(self: *ProductionGit, check: []const u8, argv: []const []const u8) !?[]u8 {
var env = try self.childEnvironment();
defer env.deinit();
const result = std.process.run(self.gpa, self.io, .{
.argv = argv,
.environ_map = &env,
.stdout_limit = .limited(preflight_output_limit_bytes),
.stderr_limit = .limited(preflight_output_limit_bytes),
}) catch |err| switch (err) {
error.StreamTooLong => return self.preflightOverran(check),
else => return err,
};
defer self.gpa.free(result.stderr);
switch (result.term) {
.exited => |code| switch (code) {
0 => return result.stdout,
1 => {
self.gpa.free(result.stdout);
return null;
},
else => {},
},
else => {},
}
self.retainDiagnostics(result.stderr);
self.gpa.free(result.stdout);
return error.GitPreflightFailed;
}
fn query(self: *ProductionGit, check: []const u8, argv: []const []const u8) ![]u8 {
return self.queryLimited(check, argv, preflight_output_limit_bytes);
}
/// Ref/root inventories are collections, not scalar preflight answers.
/// Bound them by the supported import metadata size, not a shell-line cap.
fn queryCollection(self: *ProductionGit, check: []const u8, argv: []const []const u8) ![]u8 {
return self.queryLimited(check, argv, zit.git_import_map.max_map_bytes);
}
fn queryLimited(self: *ProductionGit, check: []const u8, argv: []const []const u8, limit: u64) ![]u8 {
var env = try self.childEnvironment();
defer env.deinit();
const result = std.process.run(self.gpa, self.io, .{
.argv = argv,
.environ_map = &env,
.stdout_limit = .limited(limit),
.stderr_limit = .limited(preflight_output_limit_bytes),
}) catch |err| switch (err) {
error.StreamTooLong => {
if (limit == preflight_output_limit_bytes) return self.preflightOverran(check);
self.retainDiagnostics("git collection exceeds the supported import metadata size, or diagnostics exceed their bounded buffer\n");
return error.GitCollectionTooLarge;
},
else => return err,
};
defer self.gpa.free(result.stderr);
switch (result.term) {
.exited => |code| if (code == 0) return result.stdout,
else => {},
}
self.retainDiagnostics(result.stderr);
self.gpa.free(result.stdout);
return error.GitPreflightFailed;
}
fn terminate(context: *anyopaque) !void {
const self: *ProductionGit = @ptrCast(@alignCast(context));
const child = &(self.child orelse return);
switch (builtin.os.tag) {
.windows => {
const handle = child.id orelse return;
switch (std.os.windows.ntdll.NtTerminateProcess(handle, @fromBackingInt(@intCast(1)))) {
.SUCCESS => {},
.PROCESS_IS_TERMINATING => return error.ChildAlreadyExited,
else => return error.TerminateFailed,
}
},
.wasi => return error.UnsupportedTermination,
else => {
const pid = child.id orelse return;
std.posix.kill(pid, .KILL) catch |err| switch (err) {
error.ProcessNotFound => return error.ChildAlreadyExited,
else => return err,
};
},
}
self.exit_code = 1;
}
fn drainStderr(context: *anyopaque) !void {
const self: *ProductionGit = @ptrCast(@alignCast(context));
self.joinStderr();
}
fn joinStderr(self: *ProductionGit) void {
if (self.stderr_thread) |thread| {
thread.join();
self.stderr_thread = null;
}
}
fn retainDiagnostics(self: *ProductionGit, bytes: []const u8) void {
self.stderr_lock.lockUncancelable(self.io);
defer self.stderr_lock.unlock(self.io);
for (bytes) |byte| {
if (self.stderr_ring_len < self.stderr_ring.len) {
self.stderr_ring[(self.stderr_ring_start + self.stderr_ring_len) % self.stderr_ring.len] = byte;
self.stderr_ring_len += 1;
} else {
self.stderr_ring[self.stderr_ring_start] = byte;
self.stderr_ring_start = (self.stderr_ring_start + 1) % self.stderr_ring.len;
}
}
}
fn writeRetainedDiagnostics(self: *ProductionGit, err: *std.Io.Writer) void {
self.stderr_lock.lockUncancelable(self.io);
defer self.stderr_lock.unlock(self.io);
const retained_len = @min(
self.stderr_ring_len,
self.stderr_ring.len - cli_diagnostic_reserve_bytes,
);
var index = (self.stderr_ring_start + self.stderr_ring_len - retained_len) % self.stderr_ring.len;
for (0..retained_len) |_| {
err.writeByte(self.stderr_ring[index]) catch return;
index = (index + 1) % self.stderr_ring.len;
}
}
fn writeStartupDiagnostics(context: *anyopaque, err: *std.Io.Writer) void {
const self: *ProductionGit = @ptrCast(@alignCast(context));
self.writeRetainedDiagnostics(err);
}
fn drainStderrThread(self: *ProductionGit) void {
const file = self.stderr_file orelse return;
defer {
file.close(self.io);
self.stderr_file = null;
}
var buffer: [4096]u8 = undefined;
while (true) {
const n = file.readStreaming(self.io, &.{buffer[0..]}) catch break;
if (n == 0) break;
self.stderr_lock.lockUncancelable(self.io);
for (buffer[0..n]) |byte| {
if (self.stderr_ring_len < self.stderr_ring.len) {
self.stderr_ring[(self.stderr_ring_start + self.stderr_ring_len) % self.stderr_ring.len] = byte;
self.stderr_ring_len += 1;
} else {
self.stderr_ring[self.stderr_ring_start] = byte;
self.stderr_ring_start = (self.stderr_ring_start + 1) % self.stderr_ring.len;
}
}
self.stderr_bytes_drained += n;
self.stderr_lock.unlock(self.io);
}
}
fn writeDiagnostics(pipes: *ExportPipes, err: *std.Io.Writer, enabled: bool) void {
const self = pipes.owner orelse return;
if (!enabled) return;
self.writeRetainedDiagnostics(err);
}
fn reap(context: *anyopaque) !void {
const self: *ProductionGit = @ptrCast(@alignCast(context));
if (self.reaped) return;
self.reap_calls += 1;
// No platform may reap or clear `child` until the independently owned
// stderr descriptor is fully drained and its thread has exited.
self.joinStderr();
const child = &(self.child orelse return);
if (child.stdout) |stdout| {
stdout.close(self.io);
child.stdout = null;
}
const term = child.wait(self.io) catch |err| {
child.kill(self.io);
_ = child.wait(self.io) catch {};
self.exit_code = 1;
self.reaped = true;
self.child = null;
return err;
};
self.exit_code = switch (term) {
.exited => |code| code,
else => 1,
};
self.reaped = true;
self.child = null;
}
const stdout_reader_vtable: std.Io.Reader.VTable = .{ .stream = streamStdout };
const stderr_reader_vtable: std.Io.Reader.VTable = .{ .stream = streamStderr };
fn streamStdout(reader: *std.Io.Reader, writer: *std.Io.Writer, limit: std.Io.Limit) std.Io.Reader.StreamError!usize {
const self: *ProductionGit = @alignCast(@fieldParentPtr("stdout_buffer", @as(*[8192]u8, @ptrCast(reader.buffer.ptr))));
return streamPipe(self, reader, writer, limit, true);
}
fn streamStderr(reader: *std.Io.Reader, writer: *std.Io.Writer, limit: std.Io.Limit) std.Io.Reader.StreamError!usize {
const self: *ProductionGit = @alignCast(@fieldParentPtr("stderr_buffer", @as(*[8192]u8, @ptrCast(reader.buffer.ptr))));
return streamPipe(self, reader, writer, limit, false);
}
fn streamPipe(
self: *ProductionGit,
_: *std.Io.Reader,
writer: *std.Io.Writer,
limit: std.Io.Limit,
comptime stdout: bool,
) std.Io.Reader.StreamError!usize {
const child = &(self.child orelse return error.ReadFailed);
const file = if (stdout) child.stdout orelse return error.ReadFailed else child.stderr orelse return error.ReadFailed;
var temporary: [8192]u8 = undefined;
const requested = if (limit.toInt()) |value| @min(value, temporary.len) else temporary.len;
if (requested == 0) return 0;
const n = file.readStreaming(self.io, &.{temporary[0..requested]}) catch |err| switch (err) {
error.EndOfStream => {
if (stdout) self.stdout_reached_eof = true;
return error.EndOfStream;
},
else => return error.ReadFailed,
};
if (n == 0) {
if (stdout) self.stdout_reached_eof = true;
return 0;
}
writer.writeAll(temporary[0..n]) catch return error.WriteFailed;
return n;
}
};
/// Fault injection is deliberately at publication boundaries, rather than in
/// storage. This keeps the production storage path identical to the tested one.
pub const PublicationHooks = zit.publication.PublicationHooks;
pub const ImportLimits = struct {
/// Cumulative `data <n>` payload one import may read, raised with
/// `--max-export-payload-bytes`.
///
/// It is not a memory bound, and reading it as one misjudges both ends.
/// Payload is streamed and released per event, so resident size tracks the
/// working set, not the total: importing sqlite — a 12.2 GiB export stream
/// — held 21 MiB. What this actually bounds is the total history one
/// import may carry.
///
/// It defaulted to four GiB, which refused sqlite while over-provisioning
/// memory by three orders of magnitude. That number could not simply be
/// raised, because the per-commit file-command ceiling divided the same
/// budget: a bigger allowance for a whole history was also a bigger
/// allowance for one commit's retained command list, which unlike payload
/// is held whole. The arena now has `max_command_arena_bytes` of its own
/// and keeps the old four GiB, leaving this free to be what a total should
/// be — large enough that an ordinary repository never meets it, still
/// finite against a stream that never ends.
max_export_payload_bytes: u64 = @as(u64, 1) << 40,
/// Bytes one commit's retained file commands may occupy, which
/// `fast_export.fileCommandCeiling` restates as a command count. No CLI
/// flag raises it: at the default it admits ~104 million commands in a
/// single commit, so anything that meets it is pathological rather than
/// merely large, and the honest answer is that the importer cannot hold
/// that commit whole. It is a field rather than a constant so the limit
/// and its diagnostic stay testable from one place.
max_command_arena_bytes: u64 = @as(u64, 4) * 1024 * 1024 * 1024,
/// Bounds bytes cloned into importer state before any clone is attempted.
/// This is the ONLY bound on retained state, and it is the principled one:
/// `accountRetainedEvent` measures the bytes each event actually retains.
///
/// A second bound used to sit beside it -- a 3,000,000 cap on the ENTRY
/// COUNT -- and it was removed. It was a proxy for the same thing with an
/// arbitrary number attached, and it bound first: pkgsrc at 185,280 commits
/// over 123,843 files met it after 2h41m of translate, with 3.27 GB written
/// and well inside this budget. A ceiling that stops a legitimate import
/// before the real memory bound is reached is measuring the wrong thing.
max_retained_bytes: u64 = @as(u64, 4) * 1024 * 1024 * 1024,
/// Blob payload the importer holds for the commit that names its path.
/// Not a refusal threshold: past it the oldest held blobs are written
/// without a hint. Its own budget rather than a share of
/// `max_retained_bytes`, because it comes and goes with each commit and
/// must never be what stops an import.
max_held_blob_bytes: u64 = git_import.Importer.default_held_blob_cap,
};
/// A limit overrun names its numbers. `FileCommandLimitExceeded` alone tells an
/// operator nothing actionable: what was observed, what the ceiling was, and
/// what raises it is the entire diagnostic. Retained here and surfaced next to
/// the error name, exactly as `ProductionGit.preflightOverran` does for
/// preflight output.
pub const repair_note_max = 640;
/// The report for what the source holds that Git keeps but a check-in
/// cannot as it is: identities read as Git reads them, and file paths no
/// tree can hold, each with a count and the first, so the operator can find
/// it in Git. Null when there was neither.
pub fn repairNote(buf: *[repair_note_max]u8, importer: anytype) ?[]const u8 {
const work = importer.workCounters();
var w = std.Io.Writer.fixed(buf);
if (work.repaired_identities != 0) {
const one = work.repaired_identities == 1;
w.print("{d} identit{s} in the source {s} not NAME <EMAIL> DATE, and {s} imported as Git reads {s}; the first: {s}", .{
work.repaired_identities,
if (one) "y" else "ies",
if (one) "is" else "are",
if (one) "it was" else "each was",
if (one) "it" else "them",
importer.firstRepair() orelse "",
}) catch return null;
}
if (work.skipped_unsafe_paths != 0) {
if (w.end != 0) w.writeAll("\nimport-git: ") catch return null;
w.print("{d} file entr{s} in the source {s} a path no checkout can hold ('..', '.', an empty part, a backslash, or a name zit keeps at the root), and {s} left out; the first: '{s}'", .{
work.skipped_unsafe_paths,
if (work.skipped_unsafe_paths == 1) "y" else "ies",
if (work.skipped_unsafe_paths == 1) "has" else "have",
if (work.skipped_unsafe_paths == 1) "was" else "were",
importer.firstUnsafePath() orelse "",
}) catch return null;
}
return if (w.end == 0) null else w.buffered();
}
pub const LimitDiagnostic = struct {
buffer: [512]u8 = undefined,
len: usize = 0,
fn retain(self: *LimitDiagnostic, comptime fmt: []const u8, args: anytype) void {
const message = std.fmt.bufPrint(&self.buffer, fmt, args) catch {
// Every argument is a bounded integer, well inside the buffer; a
// truncating format would be a bug here, not a wire condition.
// Still total: report without the numbers rather than drop it.
const fallback = "import-git: a per-commit parser ceiling was exceeded; raise --max-export-payload-bytes\n";
@memcpy(self.buffer[0..fallback.len], fallback);
self.len = fallback.len;
return;
};
self.len = message.len;
}
pub fn write(self: *const LimitDiagnostic, err: *std.Io.Writer) void {
if (self.len == 0) return;
err.writeAll(self.buffer[0..self.len]) catch {};
}
/// The record a `Malformed` refusal was reading and the rule it broke.
fn retainRefusal(self: *LimitDiagnostic, reader: *const fast_export.Reader) void {
var w = std.Io.Writer.fixed(&self.buffer);
w.writeAll("import-git: the export stream was refused at ") catch {};
reader.writeRefusal(&w) catch {};
w.writeAll("\n") catch {};
self.len = w.end;
}
/// Git failed: its own message says why, and a parse refusal of the
/// stream it cut short would only mislead.
pub fn clear(self: *LimitDiagnostic) void {
self.len = 0;
}
};
pub const Dependencies = struct {
git: ChildFactory,
publication: PublicationHooks = .real_filesystem,
limits: ImportLimits = .{},
/// A test seam: corrupt the witness after the importer recorded it,
/// before the verifier reads it. Null everywhere but the corruption suite.
witness_fault: ?WitnessFault = null,
};
/// A divergence between what the import recorded writing and what it wrote,
/// that no store or staging file shows. The only way to reach the verifier's
/// tally clause with nothing a reachability pass could name.
pub const WitnessFault = enum {
/// The written tally's fingerprint moved; its count did not.
tally_drift,
/// A name no commit carried as a mirror candidate appended to the drop
/// set, after every name that belongs there.
noncandidate_drop,
};
/// The name `WitnessFault.noncandidate_drop` appends.
pub const noncandidate_drop_name = "0000000000000000000000000000000000000000000000000000000000000000";
pub const Result = @import("git_import_result.zig").Result;
pub const Request = struct {
source: []const u8,
destination_parent: std.Io.Dir,
destination_name: []const u8,
/// sync §9.1 — the new repository's project code, which every peer checks
/// in the handshake and refuses a repository without. Required, not
/// defaulted: the source is git, so there is no code to carry over, and a
/// store written without one cannot be served, cloned or adopted. The
/// caller derives it as `init` does, from the destination's absolute path.
project_code: zit.sync.HashHex,
/// `.tree = null` selects the analyzed §6.3 optimum the child factory
/// measured during preflight; `.tree = K` is the operator's explicit pin;
/// `.flat` is `--no-submanifests`, §8.1's default posture.
submanifest: git_import.SubmanifestChoice = .{ .tree = null },
/// Null when the operator chose none: committers for a fresh import, the
/// recorded choice for `--into`.
attribution: ?git_import.Attribution = null,
default_branch: ?[]const u8 = null,
migrate_default_branch: ?[]const u8 = null,
expect_default_branch: ?[]const u8 = null,
expect_imported_head: ?[]const u8 = null,
result: ?*Result = null,
json: bool = false,
quiet: bool = false,
/// Publish the import's instrumentation alongside the repository: the
/// per-object provenance map (`.zit-import-map.jsonl`) and the source ref
/// manifest (`refs`). Off by default -- a published checkout is the store
/// and its checkout record, nothing else -- and opt-in for the workflows
/// that measure imports rather than merely use them: every classifier
/// census, drop audit and git-to-zit commit lookup this project has run
/// reads the map from a PUBLISHED store, so the instrument must remain
/// obtainable, just not ambient.
provenance: bool = false,
/// Add to an existing destination from the commit map it holds, instead
/// of creating one.
into: bool = false,
/// The signature policy the launcher was given, recorded in the map by a
/// fresh import and compared by one that adds to a destination.
refuse_signed: bool = false,
/// Threads the store writes run their delta search and compression on,
/// and the verifier its sweeps, the importing thread included. Every
/// count writes the same bytes, and refuses a fault the same way, as
/// every other.
threads: usize = 1,
/// The verifier's shortest split run. Only tests lower it, so a small
/// fixture's sweeps split into several runs the way a large store's do.
verifier_min_run: usize = verifier.default_min_run,
/// Commits per window of the verifier's pooled reads; lowered only by
/// tests.
verifier_window: usize = verifier.default_checkin_window,
/// The least time between two progress lines of one phase after ingest.
/// Zero, which only tests set, narrates every unit.
progress_interval_ns: u64 = zit.progress.Meter.default_interval_ns,
};
/// The mirror store: blobs first named at a mirror-candidate path, kept out of
/// the published store until the drop set is final. It is never published,
/// so it carries a name of its own and is deleted from staging before the
/// directory is synced and renamed.
pub const mirror_store_name = zit.publication.mirror_store_name;
/// The witness's commits, spilled for the verifier. A working file of the
/// run, never published.
pub const witness_spill_name = ".zit-import-witness";
pub const published_generation_name = zit.publication.published_generation_name;
pub const PublicationBundle = zit.publication.PublicationBundle;
const LogTreeStore = zit.log_tree_store.LogTreeStore;
/// Runs one all-or-nothing import. Until `renamePreserve` succeeds, every
/// failure owns and removes only this invocation's temporary sibling.
pub fn run(
gpa: std.mem.Allocator,
io: std.Io,
request: Request,
streams: cli.Streams,
dependencies: Dependencies,
) cli.Exit {
if (request.into) return @import("git_import_into.zig").run(gpa, io, request, streams, dependencies);
var ignored_result: Result = .{};
const result = request.result orelse &ignored_result;
if (destinationExists(io, request.destination_parent, request.destination_name)) {
result.refuse("destination_exists");
streams.err.print("import-git: destination '{s}' already exists\n", .{request.destination_name}) catch {};
return .failure;
}
var pipes = dependencies.git.vtable.start_export(dependencies.git.context, request.source, null) catch |err| {
if (dependencies.git.vtable.startup_diagnostics) |write_diagnostics| {
write_diagnostics(dependencies.git.context, streams.err);
}
streams.err.print("import-git: cannot start git fast-export: {s}\n", .{@errorName(err)}) catch {};
return .failure;
};
var effective_request = request;
if (effective_request.default_branch == null) effective_request.default_branch = pipes.detected_default_branch;
var session_finished = false;
defer if (!session_finished) {
_ = finishSession(&pipes, streams, dependencies.git, request.quiet, true) catch false;
};
var publication = PublicationBundle.create(gpa, io, request.destination_parent, request.destination_name) catch |err| {
streams.err.print("import-git: cannot create temporary destination: {s}\n", .{@errorName(err)}) catch {};
return .failure;
};
defer publication.abort();
publication.publish_instrumentation = request.provenance;
var limit_diagnostic: LimitDiagnostic = .{};
var submodule_paths: u64 = 0;
importInto(
gpa,
io,
effective_request,
&publication,
&pipes,
streams,
dependencies.git,
&session_finished,
dependencies.publication,
dependencies.limits,
&limit_diagnostic,
dependencies.witness_fault,
&submodule_paths,
) catch |err| {
if (err == error.DurabilityUncertain) {
result.* = .{ .outcome = .committed_uncertain, .reason = "durability_uncertain", .changed = true, .committed = true, .durability = .uncertain };
streams.err.print("import-git: destination '{s}' was published but parent-directory durability is uncertain\n", .{request.destination_name}) catch {};
} else if (err == error.GitExportFailed) {
streams.err.writeAll("import-git: import failed: git fast-export stopped with an error; its message is above\n") catch {};
} else {
streams.err.print("import-git: import failed: {s}\n", .{@errorName(err)}) catch {};
limit_diagnostic.write(streams.err);
}
return .failure;
};
// Pre-pay the derived-state cache while the import's work is still warm.
// `Repository.open` derives `.zit-cache` on a miss, so without this the
// FIRST read verb on an imported store eats a full-store derive -- on a
// real 1M-artifact corpus that was ~342s of a 10-minute `open`. Clone has
// warmed at completion since `warmCacheFromLog` existed; import simply
// never did. After the atomic rename, against the published store, and
// best-effort like every cache write: a failure costs the first reader
// the derive it would have paid anyway.
result.success(true, null);
result.submodules.paths = submodule_paths;
if (request.destination_parent.openDir(io, request.destination_name, .{})) |published_const| {
var published = published_const;
defer published.close(io);
var workers: zit.store_workers.Pool = .{ .io = io };
workers.start(request.threads);
defer workers.deinit();
warmCache(gpa, io, request, streams, published, &workers);
if (zit.git_import_map.load(gpa, io, published)) |loaded| {
var imported = loaded;
defer imported.deinit();
result.added_checkins = imported.contents.commits.len;
} else |_| {}
} else |_| {}
if (!request.json) streams.out.print("{s}\n", .{request.destination_name}) catch return .failure;
if (!request.quiet) streams.err.print("import-git: imported '{s}'\n", .{request.source}) catch {};
return .ok;
}
/// The allocator an import runs on. Over `std.heap.smp_allocator` -- the
/// process allocator of a build without libc, such as the static Linux
/// release -- it keeps a free list of its own per power-of-two size class
/// and gives blocks back only at the end of the import.
///
/// The SMP allocator keeps its free lists per CPU slot and moves a thread to
/// the next slot whenever one size class runs dry in the current one. A
/// single-threaded import does exactly that, constantly, so the blocks one
/// commit frees land in one slot while the next commit allocates fresh
/// slabs in another. On a four-core host that left a long import holding
/// several times its live memory resident, growing with the history. Here a
/// freed block is reused for the next request of its class wherever it was
/// freed, so resident memory follows each class's peak.
///
/// Blocks are requested from the backing allocator at exactly their class
/// size and alignment, which is how the SMP allocator sizes them anyway, so
/// nothing is lost to the extra layer, and a block that escapes the import
/// can still be freed straight to the backing allocator. Requests past the
/// SMP allocator's largest class are whole mappings there and pass through.
/// Over any other allocator the layer is a pass-through.
const ImportHeap = struct {
backing: std.mem.Allocator,
mutex: std.atomic.Mutex = .unlocked,
free_lists: [class_count]?*FreeBlock = @splat(null),
const FreeBlock = struct { next: ?*FreeBlock };
const min_shift = std.math.log2_int(usize, @sizeOf(FreeBlock));
/// The SMP allocator's largest small class: 32 KiB.
const max_shift = 15;
const class_count = max_shift - min_shift + 1;
fn allocator(self: *ImportHeap) std.mem.Allocator {
if (self.backing.vtable != std.heap.smp_allocator.vtable) return self.backing;
return self.caching();
}
fn caching(self: *ImportHeap) std.mem.Allocator {
return .{ .ptr = self, .vtable = &.{ .alloc = alloc, .resize = resize, .remap = remap, .free = free } };
}
fn deinit(self: *ImportHeap) void {
for (&self.free_lists, min_shift..) |*list, shift| {
while (list.*) |block| {
list.* = block.next;
const bytes: [*]u8 = @ptrCast(block);
self.backing.rawFree(bytes[0 .. @as(usize, 1) << @intCast(shift)], .fromByteUnits(@as(usize, 1) << @intCast(shift)), @returnAddress());
}
}
self.* = undefined;
}
/// The class `len` bytes at `alignment` fall in, or null past the largest.
fn shiftOf(len: usize, alignment: std.mem.Alignment) ?usize {
const shift = @max(std.math.log2_int_ceil(usize, @max(len, 1)), @as(usize, @backingInt(alignment)), min_shift);
return if (shift > max_shift) null else shift;
}
fn lock(self: *ImportHeap) void {
while (!self.mutex.tryLock()) std.atomic.spinLoopHint();
}
fn alloc(context: *anyopaque, len: usize, alignment: std.mem.Alignment, ra: usize) ?[*]u8 {
const self: *ImportHeap = @ptrCast(@alignCast(context));
const shift = shiftOf(len, alignment) orelse return self.backing.rawAlloc(len, alignment, ra);
self.lock();
defer self.mutex.unlock();
const list = &self.free_lists[shift - min_shift];
if (list.*) |block| {
list.* = block.next;
return @ptrCast(block);
}
return self.backing.rawAlloc(@as(usize, 1) << @intCast(shift), .fromByteUnits(@as(usize, 1) << @intCast(shift)), ra);
}
fn resize(context: *anyopaque, memory: []u8, alignment: std.mem.Alignment, new_len: usize, ra: usize) bool {
const self: *ImportHeap = @ptrCast(@alignCast(context));
const old_shift = shiftOf(memory.len, alignment);
const new_shift = shiftOf(new_len, alignment);
if (old_shift == null and new_shift == null) return self.backing.rawResize(memory, alignment, new_len, ra);
return old_shift != null and old_shift == new_shift;
}
fn remap(context: *anyopaque, memory: []u8, alignment: std.mem.Alignment, new_len: usize, ra: usize) ?[*]u8 {
const self: *ImportHeap = @ptrCast(@alignCast(context));
const old_shift = shiftOf(memory.len, alignment);
const new_shift = shiftOf(new_len, alignment);
if (old_shift == null and new_shift == null) return self.backing.rawRemap(memory, alignment, new_len, ra);
return if (old_shift != null and old_shift == new_shift) memory.ptr else null;
}
fn free(context: *anyopaque, memory: []u8, alignment: std.mem.Alignment, ra: usize) void {
const self: *ImportHeap = @ptrCast(@alignCast(context));
const shift = shiftOf(memory.len, alignment) orelse return self.backing.rawFree(memory, alignment, ra);
self.lock();
defer self.mutex.unlock();
const block: *FreeBlock = @ptrCast(@alignCast(memory.ptr));
const list = &self.free_lists[shift - min_shift];
block.next = list.*;
list.* = block;
}
};
/// Said once per import that met a gitlink: a checkout holds the ledger,
/// not the submodule, so the operator hears it here rather than from an
/// empty directory later.
/// Said when squash merges were linked to the pull requests they took.
pub const squash_note = "import-git: {d} squash merge(s) recorded as cherry-picks of the pull request each took\n";
pub const submodule_note = "import-git: {d} submodule path(s) pinned in .zit-modules rather than the tree; " ++
"the pinned commits live in other repositories and are not imported, and .gitmodules was kept as a plain file\n";
fn importInto(
process_gpa: std.mem.Allocator,
io: std.Io,
request: Request,
publication: *PublicationBundle,
pipes: *ExportPipes,
streams: cli.Streams,
child: ChildFactory,
session_finished: *bool,
hooks: PublicationHooks,
limits: ImportLimits,
diagnostic: *LimitDiagnostic,
witness_fault: ?WitnessFault,
submodule_paths_out: *u64,
) !void {
var heap: ImportHeap = .{ .backing = process_gpa };
defer heap.deinit();
const gpa = heap.allocator();
const temporary = publication.temporary;
try hooks.check(.create_store);
// The published store, written in place. Every artifact is delta-searched
// and compressed once, as it is written, into the file the rename ships:
// blobs against the previous version of their path, sub-manifest nodes
// against the node they replace, check-ins against their parent.
var store_file = try publication.createPublishedGeneration();
var store_file_open = true;
defer if (store_file_open) store_file.close(io);
var file_storage = storage.FileStorage.init(io, store_file, 0);
// §7.1/§7.2 — this byte is a `HashPolicy` ordinal, NOT a `HashAlgo`. The two
// enums overlap numerically and mean different things: `HashAlgo` is
// {sha1=0, sha3_256=1}, `HashPolicy` is {sha1=0, auto=1, sha3=2, ...}. This
// wrote `@intFromEnum(HashAlgo.sha1)`, so every imported repository
// advertised the policy `sha1` by coincidence of ordinal — and the obvious
// "switch it to sha3_256" repair would have yielded 1 = `auto`, silently
// arming §7.4's promotion ratchet instead. Derived, never a literal.
//
// The import mints SHA-1 names throughout, so §7.2's rule gives `auto`:
// a store that holds SHA-1 names and no SHA3 ones. When the naming
// algorithm learns to follow its source, this call needs no edit — the
// predicate changes, the derivation does not.
const import_hash_policy: u8 = @backingInt(importDefaultPolicy());
var log = try logstore.LogStore.open(gpa, file_storage.storageFile(), import_hash_policy);
var log_live = true;
defer if (log_live) log.deinit(gpa);
// The writer's own record of every name it puts into the published store,
// which the verifier compares with what the closed file's index holds. The
// key is drawn per run and never stored, so no repository content can be
// chosen to make two different name sets tally alike.
var tally_key: [16]u8 = undefined;
io.randomSecure(&tally_key) catch return error.EntropyUnavailable;
var tally = logstore.NameTally.init(tally_key);
log.written = &tally;
// The mirror store. A blob first named at a mirror-candidate path is
// written here instead: whether it is dropped is decided by every commit
// that names it, so it cannot go into the published store until the drop
// set is final. A repository without Fossil mirror metadata leaves this
// an empty log.
try hooks.check(.create_mirror_store);
var mirror_file = try publication.createMirrorStore();
var mirror_file_open = true;
defer if (mirror_file_open) mirror_file.close(io);
var mirror_storage = storage.FileStorage.init(io, mirror_file, 0);
var mirror = try logstore.LogStore.open(gpa, mirror_storage.storageFile(), import_hash_policy);
var mirror_live = true;
defer if (mirror_live) mirror.deinit(gpa);
// Both stores hand their batches' delta search and compression to the
// same threads; the stores are written one at a time, by this thread.
// Once both are frozen, the verifier spreads its sweeps over them too.
var workers: zit.store_workers.Pool = .{ .io = io };
workers.start(request.threads);
defer workers.deinit();
log.workers = &workers;
mirror.workers = &workers;
var tree_store = LogTreeStore{ .gpa = gpa, .log = &log };
defer tree_store.deinitCache();
// One shape for the whole import, resolved before the first tree: flat when
// the operator declined sub-manifests, else the operator's override if
// given, else preflight's analyzed §6.3 optimum, else `resolveOptions`' own
// default when neither is available.
//
// The analysis itself runs either way. Skipping it under `.flat` would make
// tree emission the SECOND thing the two modes differ in -- the first being
// a `git ls-tree` of the whole tip tree during preflight -- and the reason
// this flag exists is to give a benchmark two imports that differ in
// nothing else. One discarded measurement is the cheaper wart, and it is
// not a new one: an explicit `--submanifest-k` already discards it.
const resolved_choice: git_import.SubmanifestChoice = switch (request.submanifest) {
.flat => .flat,
.tree => |pin| .{ .tree = pin orelse pipes.analyzed_submanifest_k },
};
const shape = (try git_import.resolveOptions(.{ .submanifest = resolved_choice })).shape;
const pinned_k: u4 = switch (shape) {
// Not a k. `Settings.pinned()` reads zero as "no pin", which is the
// truthful state for a repository that emitted no sub-manifest.
.flat => 0,
.tree => |tree| tree.k,
};
switch (request.submanifest) {
// Silence is the symmetric behavior for "do the default thing": a flat
// import has no k to report, so it reports nothing.
.flat => {},
.tree => |pin| if (!request.quiet and pin == null and pipes.analyzed_submanifest_k != null) {
streams.err.print("import-git: analyzed sub-manifest k={d}\n", .{pinned_k}) catch {};
},
}
var importer = try git_import.Importer.init(gpa, &log, tree_store.treeStore(), .{
.submanifest = switch (shape) {
.flat => .flat,
.tree => |tree| .{ .tree = tree.k },
},
.attribution = request.attribution orelse .committer,
});
var importer_live = true;
defer if (importer_live) importer.deinit();
importer.keepMirrorCandidatesIn(&mirror);
importer.setHeldBlobCap(limits.max_held_blob_bytes);
if (pipes.squash_picks) |picks| importer.setSquashPicks(&picks.by_commit);
var reader = fast_export.Reader.init(gpa, pipes.stdout, .{
.max_line = max_fast_export_line_bytes,
.max_data = 1024 * 1024 * 1024,
.max_path = 4096,
// No event ceiling. `git fast-export` emits one event per blob,
// commit, tag and reset, so the count scales with the source repo's
// history rather than with anything this import spends: pkgsrc's
// 185,280 commits are already a fifth of the 1,000,000 records this
// used to admit, before a single blob event is counted. Memory is
// bounded by bytes on both sides -- `max_total_data` for payload and
// `max_retained_bytes` for retained state -- and those bound the
// thing that actually costs. A stream of endless empty records can
// still spin, but it retains nothing and stays interruptible.
.max_events = std.math.maxInt(u64),
.max_total_data = limits.max_export_payload_bytes,
.max_command_arena_bytes = limits.max_command_arena_bytes,
});
defer reader.deinit();
consumeExport(&reader, &importer, limits, diagnostic, .{
.streams = streams,
.io = io,
.quiet = request.quiet,
.commit_total = pipes.commit_total,
}) catch |err| {
const scripted_child_failed = pipes.exit_code_ref == null and terminalExitCode(pipes) != 0;
const child_finished_before_cancellation = finishSession(pipes, streams, child, request.quiet, true) catch |lifecycle_err| {
session_finished.* = true;
return lifecycle_err;
};
session_finished.* = true;
if (scripted_child_failed or (child_finished_before_cancellation and terminalExitCode(pipes) != 0)) {
diagnostic.clear();
return error.GitExportFailed;
}
return err;
};
_ = finishSession(pipes, streams, child, request.quiet, false) catch |err| {
session_finished.* = true;
return err;
};
session_finished.* = true;
if (terminalExitCode(pipes) != 0) return error.GitExportFailed;
var finalized = importer.finalizeRefs(.{ .default_branch = request.default_branch }) catch |err| {
if (err == error.CanonicalRefCollision) if (importer.ref_collision) |collision| {
diagnostic.retain(
"import-git: '{s}' and '{s}' both name 'refs/{s}'; rename one in the source repository, or exclude it, and re-run\n",
.{ collision.first_source, collision.second_source, collision.canonical },
);
};
return err;
};
defer finalized.deinit(gpa);
for (finalized.controls) |control| try putSha1Artifact(&log, control);
// The strongest statement this code can make about a flat import, and it
// costs nothing: `stored_submanifest` is `emitted_nodes > 0`, a whole-import
// fact derived from work actually done, computed without consulting the
// shape -- not "the flag was set", which is what an assertion on `shape`
// would say.
//
// `emitted_nodes` counts nodes this import CREATED, so on its own it would
// leave a root built entirely from reused nodes uncovered. It is not
// uncovered here, because the published store is created by this run a
// few dozen lines above (`createPublishedGeneration`, an exclusive create):
// reuse can only hit a node this same import already emitted, so zero
// emitted implies zero reused implies no S reference anywhere.
//
// A returned error rather than an assert because this aborts a
// user-visible operation and must unwind the transaction so DEST is never
// published.
if (std.meta.activeTag(shape) == .flat and finalized.stored_submanifest) {
return error.SubmanifestEmittedUnderFlatImport;
}
// The drop set, captured while the importer is still alive.
//
// `finalized.ledger.candidates[].name` BORROWS from the importer, which is
// deinitialized at the freeze below. `mirrorDropNames` hands back owned
// copies of exactly the same set, so this is the only capture that
// outlives the importer.
try hooks.check(.capture_drop_set);
var drops = try MirrorDropSet.capture(gpa, &importer);
defer drops.deinit(gpa);
// The mirror store's survivors join the published store. The drop set is
// the classifier's, and only the classifier's -- every name in it has a
// counted zero retained-reference, so nothing reachable from an emitted
// tree can be in it. Done before the map is written, so the artifact
// count the map records is the published store's.
try hooks.check(.copy_mirror_survivors);
const dropped = try copyMirrorSurvivors(gpa, &mirror, &log, &drops.set);
// The classifier authorized a set; the copy reports what it withheld.
// Equal by construction -- a candidate reaches the drop set only when
// nothing but candidate paths ever named it, and such a blob is written to
// the mirror store and nowhere else -- which is exactly why the two being
// unequal means the classifier and the mirror store disagree.
if (dropped != drops.set.count()) return error.MirrorDropSetNotApplied;
try log.putConfig(.{
// Same §7.2 derivation as the header above, and the same reason it is
// not a literal: this field is a `HashPolicy`, and the `HashAlgo`
// ordinal that used to sit here only looked right.
.hash_policy = @backingInt(importDefaultPolicy()),
.default_branch = finalized.primary_branch,
.project_id = "",
.project_code = &request.project_code,
// These two are a pair, not two independent settings. Under `.flat`,
// `submanifest_k` is deliberately zero -- outside `artifact.min_k ..
// max_k` -- and that is only safe while `enable_submanifests` is false,
// because `Settings.emission()` short-circuits on disabled before it
// can range-check the pin. Enabled with a zero pin is the
// `.out_of_range` misconfiguration, so the two must move together.
.submanifest_k = pinned_k,
.enable_submanifests = std.meta.activeTag(shape) == .tree,
// The latch is one-way and means "an S reference is stored here", not
// "trees are on" — see its doc in logstore.zig. Leaving it false while
// the store holds S-bearing artifacts is a lie that stays hidden only
// while `enable_submanifests` is true, because the §7.2 gate is the
// disjunction of the two: a later `zit submanifest disable` would flip
// the repository to advertising avoid-submanifests while holding trees.
.seen_submanifest = finalized.stored_submanifest,
});
try writeRefs(gpa, io, temporary, finalized.refs);
try writeMap(gpa, io, temporary, request, child, &importer, &finalized, pinned_k, shape, log.artifactCount());
try checkout.write(io, temporary, .{
// The imported store is complete, but no worktree files have been
// materialized. Publishing the default tip here would make `update`
// incorrectly treat that unmapped worktree as current.
.version = "",
.branch = finalized.primary_branch,
.merge_parent = "",
.special_op = "",
.manifest_flags = "",
});
// What the published store is verified AGAINST, captured for the same
// reason and at the same moment as the drop set. `finalized.ledger` cannot
// serve: every string in it borrows the importer, which the freeze below
// destroys. The tally is taken last: nothing writes an artifact after it.
// Its commits grow with the history, so they go to a working file of the
// run, under a key drawn here and never stored.
var witness_key: git_import.witness_spill.Key = undefined;
io.randomSecure(&witness_key) catch return error.EntropyUnavailable;
var witness = witness: {
var file = try temporary.createFile(io, witness_spill_name, .{ .exclusive = true });
defer file.close(io);
var buffer: [64 * 1024]u8 = undefined;
var writer = file.writerStreaming(io, &buffer);
var captured = importer.captureWitness(gpa, finalized, &writer.interface, witness_key) catch |err| switch (err) {
error.WriteFailed => return writer.err orelse error.WriteFailed,
else => |other| return other,
};
errdefer captured.deinit();
writer.interface.flush() catch return writer.err orelse error.WriteFailed;
break :witness captured;
};
defer witness.deinit();
log.written = null;
witness.written = tally;
if (witness_fault) |fault| switch (fault) {
.tally_drift => witness.written.?.sum +%= 1,
.noncandidate_drop => {},
};
var faulted_drops: std.ArrayList([]const u8) = .empty;
defer faulted_drops.deinit(gpa);
if (witness_fault == .noncandidate_drop) {
try faulted_drops.appendSlice(gpa, drops.names);
try faulted_drops.append(gpa, noncandidate_drop_name);
}
const drop_names: []const []const u8 = if (witness_fault == .noncandidate_drop) faulted_drops.items else drops.names;
var map = try temporary.createFile(io, ".zit-import-map.jsonl", .{ .read = true, .exclusive = true });
var map_open = true;
defer if (map_open) map.close(io);
try hooks.check(.before_flush_map);
{
// Streamed from the importer's records: on a long history the text
// runs to hundreds of megabytes, and nothing needs it whole.
var map_buf: [64 * 1024]u8 = undefined;
var map_writer = map.writerStreaming(io, &map_buf);
importer.writeProvenance(&finalized, &map_writer.interface) catch |err| switch (err) {
error.WriteFailed => return map_writer.err orelse error.WriteFailed,
else => |other| return other,
};
map_writer.interface.flush() catch return map_writer.err orelse error.WriteFailed;
}
try hooks.check(.after_flush_map);
try hooks.check(.before_flush_store);
try log.commit();
try hooks.check(.after_flush_store);
try hooks.check(.before_sync_store);
try store_file.sync(io);
try hooks.check(.after_sync_store);
try hooks.check(.before_sync_map);
try map.sync(io);
try hooks.check(.after_sync_map);
// The verifier reads the mirror store as evidence for the drop set, so it
// is made as durable as the store it is evidence about.
try hooks.check(.before_sync_mirror_store);
try mirror.commit();
try mirror_file.sync(io);
try hooks.check(.after_sync_mirror_store);
// Freeze: no store and no map remains writable after this point.
try hooks.check(.before_close_store);
// Read before the importer is gone; reported after verification, where a
// completion statement is finally safe to make.
const sterilized_blobs = importer.workCounters().sterilized_blobs;
const submodule_paths = importer.submodulePathCount();
const squash_picks = importer.squashPicksWritten();
var repair_buf: [repair_note_max]u8 = undefined;
const repair_note = repairNote(&repair_buf, importer);
const store_bytes = log.size();
importer.deinit();
importer_live = false;
log.deinit(gpa);
log_live = false;
store_file.close(io);
store_file_open = false;
try hooks.check(.after_close_store);
try hooks.check(.before_close_mirror_store);
mirror.deinit(gpa);
mirror_live = false;
mirror_file.close(io);
mirror_file_open = false;
try hooks.check(.after_close_mirror_store);
try hooks.check(.before_close_map);
map.close(io);
map_open = false;
try hooks.check(.after_close_map);
try publication.advance(.frozen);
// The operator has watched commits count up for the whole ingest; this
// phase line keeps the narration alive through the store-wide sweep that
// follows, which is the stretch that otherwise reads as a hang.
if (!request.quiet) streams.err.print("import-git: ingest complete, verifying\n", .{}) catch {};
try verifyFrozen(gpa, io, temporary, &witness, drop_names, streams, hooks, request.quiet, &workers, request.verifier_min_run, request.verifier_window, progressOf(request, io, streams));
// Read by nothing after the verifier, and never published.
try temporary.deleteFile(io, witness_spill_name);
try publication.advance(.verified);
// Printed only once nothing further can invalidate it. Ahead of
// verification this reads as a completion statement on a run that may still
// fail, which is the one reading an operator must not get from it.
if (!request.quiet) streams.err.print(
"import-git: wrote {d} artifacts in {d} bytes (dropped {d} mirror artifacts)\n",
.{ tally.count, store_bytes, dropped },
) catch {};
// Said at import because a checkout of these files differs from the
// source's by the appended line, and the operator should not first meet
// that as a diff with nothing connecting it back to the import.
if (!request.quiet and sterilized_blobs != 0) streams.err.print(
"import-git: {d} tracked file(s) hold a Fossil artifact; each was stored with Fossil's " ++
"sterilizing line appended so it stays file content, and export-git removes the line\n",
.{sterilized_blobs},
) catch {};
if (!request.quiet and submodule_paths != 0) streams.err.print(submodule_note, .{submodule_paths}) catch {};
if (!request.quiet and squash_picks != 0) streams.err.print(squash_note, .{squash_picks}) catch {};
submodule_paths_out.* = submodule_paths;
if (!request.quiet) if (repair_note) |note| streams.err.print("import-git: {s}\n", .{note}) catch {};
if (!request.quiet) {
streams.err.print("import-git: verified, publishing\n", .{}) catch {};
streams.err.flush() catch {};
}
// The rename ships the staging DIRECTORY, so the mirror store has to leave
// it before the commit point or it publishes alongside the store.
try hooks.check(.before_discard_mirror_store);
try publication.discardMirrorStore();
try publication.discardInstrumentation();
try hooks.check(.after_discard_mirror_store);
try publication.verifyStagingManifest();
try hooks.check(.before_sync_temporary_directory);
try storage.syncDirectory(io, temporary);
try hooks.check(.after_sync_temporary_directory);
try publication.publish(request.destination_name, hooks);
// If this final durability boundary fails, the destination is intentionally
// retained: its content was fully verified and atomically published.
hooks.check(.sync_parent_after_publish) catch return error.DurabilityUncertain;
storage.syncDirectory(io, publication.parent) catch return error.DurabilityUncertain;
}
pub fn consumeExport(
reader: *fast_export.Reader,
importer: *git_import.Importer,
limits: ImportLimits,
diagnostic: *LimitDiagnostic,
progress: Progress,
) !void {
var retained_bytes: u64 = 0;
var reporter = ProgressReporter.init(progress);
while (reader.next() catch |err| switch (err) {
// Reader admits events and cumulative payload bytes before allocating
// an event arena or payload, so its exact bound failures map directly.
// This import sets no event ceiling, so the reader cannot raise this
// one; the mapping stays because the reader can still express it and a
// silent mistranslation would be worse than an arm that never runs.
error.EventLimitExceeded => return error.ExportEventLimitExceeded,
error.DataLimitExceeded => {
const overrun = reader.payload_overrun.?;
switch (overrun.bound) {
.cumulative_total => diagnostic.retain(
"import-git: the export stream's cumulative payload exceeded its budget: observed at least {d} bytes against a budget of {d} bytes; parsing stopped there, so the stream's full payload is larger than the observed total; re-run with --max-export-payload-bytes above the stream's full payload\n",
.{ overrun.observed_at_least, overrun.ceiling },
),
// No flag reaches this ceiling, so naming one would be a lie.
.single_payload => diagnostic.retain(
"import-git: a single export payload of {d} bytes exceeded the {d}-byte per-payload ceiling; no option raises this ceiling\n",
.{ overrun.observed_at_least, overrun.ceiling },
),
}
return error.ExportPayloadLimitExceeded;
},
error.FileCommandLimitExceeded => {
const overrun = reader.file_command_overrun.?;
diagnostic.retain(
"import-git: a commit exceeded the per-commit file-command ceiling: observed at least {d}, ceiling {d}, derived from the {d}-byte per-commit arena budget at {d} minimum retained bytes per command. This is a structural bound on one commit, not a total: no option raises it, because a commit at this width is beyond what the importer can hold whole\n",
.{
overrun.observed_at_least,
overrun.ceiling,
overrun.arena_budget_bytes,
overrun.min_bytes_per_command,
},
);
return error.ExportFileCommandLimitExceeded;
},
error.Malformed => {
diagnostic.retainRefusal(reader);
return err;
},
else => return err,
}) |lease| {
var event = lease;
defer event.deinit();
try accountRetainedEvent(&retained_bytes, event.value, limits.max_retained_bytes);
try importer.ingest(event.value);
if (event.value == .commit) reporter.commitIngested();
}
}
/// What the ingest loop needs to narrate itself: where to write, whether the
/// operator asked for silence, and the preflight's commit count when it took
/// one.
const Progress = struct {
streams: cli.Streams,
io: std.Io,
quiet: bool,
commit_total: ?u64,
};
/// Strided progress for the hours-long phase. One stderr line at most every
/// two seconds, so a pkgsrc-sized import narrates itself a few hundred times
/// rather than 185,000 -- and a short import usually says nothing at all.
const ProgressReporter = struct {
progress: Progress,
commits: u64 = 0,
reported_at: u64 = 0,
reported_at_ns: i96 = 0,
const report_interval_ns: u64 = 2 * std.time.ns_per_s;
// Clock reads go through a syscall, so they are strided the same way the
// lines are: one read per checked batch, not one per commit.
const check_stride: u64 = 64;
fn init(progress: Progress) ProgressReporter {
return .{
.progress = progress,
.reported_at_ns = if (progress.quiet) 0 else std.Io.Clock.now(.awake, progress.io).nanoseconds,
};
}
fn commitIngested(self: *ProgressReporter) void {
self.commits += 1;
if (self.progress.quiet) return;
if (self.commits % check_stride != 0) return;
const now_ns = std.Io.Clock.now(.awake, self.progress.io).nanoseconds;
const elapsed_ns: u64 = @intCast(@max(now_ns - self.reported_at_ns, 1));
if (elapsed_ns < report_interval_ns) return;
self.reported_at_ns = now_ns;
const rate = (self.commits - self.reported_at) * std.time.ns_per_s / elapsed_ns;
self.reported_at = self.commits;
const err = self.progress.streams.err;
if (self.progress.commit_total) |total| {
err.print("import-git: {d}/{d} commits ({d}%), {d}/s\n", .{
self.commits,
total,
self.commits * 100 / @max(total, 1),
rate,
}) catch {};
} else {
err.print("import-git: {d} commits, {d}/s\n", .{ self.commits, rate }) catch {};
}
err.flush() catch {};
}
};
const sha1_artifact_name_bytes: u64 = std.crypto.hash.Sha1.digest_length * 2;
fn accountRetainedEvent(total: *u64, event: fast_export.Event, limit: u64) !void {
switch (event) {
.blob => |blob| {
try accountRetainedFixed(total, @as(usize, 2) + @intFromBool(blob.original_oid != null), limit);
if (blob.original_oid) |oid| try accountRetainedCopies(total, oid, 2, limit);
},
.commit => |commit| {
try accountRetainedFixed(total, @as(usize, 3) + @intFromBool(commit.original_oid != null) + @intFromBool(commit.from != null), limit);
if (commit.original_oid) |oid| try accountRetainedCopies(total, oid, 2, limit);
try accountRetainedCopies(total, commit.ref, 3, limit);
try accountRetainedIdentity(total, commit.author, limit);
try accountRetainedIdentity(total, commit.committer, limit);
try accountRetainedSlice(total, commit.message, limit);
try accountRetainedPaths(total, commit.commands, limit);
},
.reset => |reset| {
try accountRetainedCopies(total, reset.ref, 2, limit);
if (reset.target != null) try accountRetainedFixed(total, 1, limit);
},
.tag => |tag| {
const source_len = std.math.add(u64, @as(u64, "refs/tags/".len), std.math.cast(u64, tag.name.len) orelse return error.ImporterRetainedByteLimitExceeded) catch return error.ImporterRetainedByteLimitExceeded;
try accountRetainedLengthCopies(total, source_len, 4, limit);
try accountRetainedSlice(total, tag.name, limit);
if (tag.original_oid) |oid| try accountRetainedSlice(total, oid, limit);
try accountRetainedFixed(total, @as(usize, 2) + @intFromBool(tag.mark != null), limit);
try accountRetainedIdentity(total, tag.tagger, limit);
try accountRetainedSlice(total, tag.message, limit);
},
.done => {},
}
}
/// Returns the exact retained-state allocation charge for one parsed event.
/// The import loop and limit-boundary tests share this calculation so a new
/// retained field cannot silently create a guessed budget.
pub fn retainedByteCharge(event: fast_export.Event) !u64 {
var total: u64 = 0;
try accountRetainedEvent(&total, event, std.math.maxInt(u64));
return total;
}
fn accountRetainedIdentity(total: *u64, identity: fast_export.Identity, limit: u64) !void {
try accountRetainedSlice(total, identity.name, limit);
try accountRetainedSlice(total, identity.email, limit);
}
fn accountRetainedPaths(total: *u64, commands: []const fast_export.FileCommand, limit: u64) !void {
for (commands) |command| switch (command) {
.modify => |modify| try accountRetainedSlice(total, modify.path, limit),
.delete => |path| try accountRetainedSlice(total, path, limit),
.rename => |rename| {
try accountRetainedSlice(total, rename.old, limit);
try accountRetainedSlice(total, rename.new, limit);
},
.copy => |copy| {
try accountRetainedSlice(total, copy.old, limit);
try accountRetainedSlice(total, copy.new, limit);
},
.delete_all => {},
};
}
fn accountRetainedFixed(total: *u64, copies: usize, limit: u64) !void {
try accountRetainedLengthCopies(total, sha1_artifact_name_bytes, copies, limit);
}
fn accountRetainedCopies(total: *u64, bytes: []const u8, copies: usize, limit: u64) !void {
const len = std.math.cast(u64, bytes.len) orelse return error.ImporterRetainedByteLimitExceeded;
try accountRetainedLengthCopies(total, len, copies, limit);
}
fn accountRetainedSlice(total: *u64, bytes: []const u8, limit: u64) !void {
try accountRetainedCopies(total, bytes, 1, limit);
}
fn accountRetainedLengthCopies(total: *u64, len: u64, copies: usize, limit: u64) !void {
var remaining = copies;
while (remaining != 0) : (remaining -= 1) {
if (total.* > limit or len > limit - total.*) return error.ImporterRetainedByteLimitExceeded;
total.* += len;
}
}
/// The exact candidate-only drop filter, owned until verification is done.
///
/// Two impedance mismatches meet here and both are load-bearing. The classifier
/// reports HEX artifact names; `pack.NameSet` holds RAW digest bytes, so a
/// hex string handed straight through would match nothing and the copy would
/// withhold nothing while reporting success.
/// And `pack.NameSet` BORROWS its keys, so the decoded digests need storage
/// that outlives the importer they came from. This owns that storage and
/// nothing else.
const MirrorDropSet = struct {
/// One stable cell per name. `set`'s keys slice into these, so this must be
/// allocated once at its final length and never resized.
digests: [][32]u8 = &.{},
set: pack.NameSet = .{},
/// The same set in the hex spelling the verifier's name-set difference
/// compares against. Owned here rather than re-derived: `digests` is a
/// fixed-width cell array whose entries do not carry their own length, so
/// hex could not be recovered from it without also carrying the widths.
names: [][]u8 = &.{},
fn capture(gpa: std.mem.Allocator, importer: *const git_import.Importer) !MirrorDropSet {
const names = try importer.mirrorDropNames(gpa);
var owned = true;
defer if (owned) {
for (names) |name| gpa.free(name);
gpa.free(names);
};
var self: MirrorDropSet = .{};
errdefer self.deinit(gpa);
self.digests = try gpa.alloc([32]u8, names.len);
// `names` is repository-derived, and bounded: it is a subset of the
// importer's mirror-candidate keys, which are retained state and so sit
// under `limits.max_retained_bytes`, the budget that bounds everything
// the importer holds. (An entry-count ceiling used to be named here too;
// it was removed for measuring the wrong thing -- see that field's
// note.) Keys of a hash map, so also distinct: `set.count()`
// below equals `names.len`, which is what lets the caller compare the
// set's size to the count the mirror-store copy withheld.
for (names, 0..) |name, index| {
// A name the classifier authorized dropping but that will not
// decode is a contradiction in this importer's own output, not an
// operator error -- and the safe reading of "I cannot tell which
// artifact this is" is to abort the import, never to copy on with
// a filter that is quietly narrower than the one that was proven.
const raw = rawHashName(name, &self.digests[index]) orelse return error.MirrorDropNameMalformed;
self.set.add(gpa, raw) catch |err| switch (err) {
error.BadKey => return error.MirrorDropNameMalformed,
error.OutOfMemory => return error.OutOfMemory,
};
}
self.names = names;
owned = false;
return self;
}
fn deinit(self: *MirrorDropSet, gpa: std.mem.Allocator) void {
self.set.deinit(gpa);
gpa.free(self.digests);
for (self.names) |name| gpa.free(name);
gpa.free(self.names);
self.* = undefined;
}
};
/// Copy every mirror-store artifact the drop set does not name into the
/// published store, in the mirror store's persisted order, and return how many
/// were withheld. With an empty drop set and an empty mirror store -- every
/// repository without Fossil mirror metadata -- this reads nothing.
///
/// The bytes are read back through the mirror store's own verified reader, so
/// a frame that does not reconstruct stops the import here rather than being
/// carried into the published store. Each survivor is written the way any
/// artifact is, delta-searched against the published store's recent window.
fn copyMirrorSurvivors(
gpa: std.mem.Allocator,
mirror: *logstore.LogStore,
log: *logstore.LogStore,
drops: *const pack.NameSet,
) !usize {
const rids = try mirror.ridsInPersistedOrder(gpa);
defer gpa.free(rids);
var withheld: usize = 0;
for (rids) |rid| {
const digest = mirror.hashOf(rid);
if (drops.contains(digest)) {
withheld += 1;
continue;
}
const bytes = (try mirror.contentUnmasked(gpa, rid, null)) orelse return error.MirrorStoreCorrupt;
defer gpa.free(bytes);
_ = try log.putArtifact(digest, bytes, null);
}
return withheld;
}
/// The complete pre-publication verification, over the frozen published
/// store, the frozen mirror store, and everything staging holds beside them.
///
/// Both stores open READ-ONLY. `openReadOnly` is not merely the polite choice
/// here: `LogStore.open` is the append/repair path, and on a zero-length or
/// torn-header file it writes a header or truncates -- which is the one thing a
/// verifier must never do to the evidence it is judging. It also takes the
/// store's own recorded hash kind rather than assuming one: a mismatched
/// header makes every artifact in the store unaddressable.
///
/// The clause pipeline itself lives in `git_import_verify.zig`. This function
/// is the transaction's half of it: open the files, read the three staging
/// artifacts, and translate a refusal into an aborted publication.
fn verifyFrozen(
gpa: std.mem.Allocator,
io: std.Io,
temporary: std.Io.Dir,
witness: *const git_import.ImportWitness,
drop_names: []const []const u8,
streams: cli.Streams,
hooks: PublicationHooks,
quiet: bool,
workers: *zit.store_workers.Pool,
min_run: usize,
checkin_window: usize,
progress: ?verifier.Progress,
) !void {
try hooks.check(.verify_mirror_store);
var mirror_file = temporary.openFile(io, mirror_store_name, .{ .mode = .read_only }) catch |e|
return reportUnreadable(streams, .mirror_store_unreadable, mirror_store_name, e);
defer mirror_file.close(io);
const mirror_size = (mirror_file.stat(io) catch |e|
return reportUnreadable(streams, .mirror_store_unreadable, mirror_store_name, e)).size;
var mirror_storage = storage.FileStorage.init(io, mirror_file, mirror_size);
var mirror_log = logstore.LogStore.openReadOnly(gpa, mirror_storage.storageFile()) catch |e|
return reportUnreadable(streams, .mirror_store_unreadable, mirror_store_name, e);
defer mirror_log.deinit(gpa);
try hooks.check(.verify_store);
var store_file = temporary.openFile(io, published_generation_name, .{ .mode = .read_only }) catch |e|
return reportUnreadable(streams, .published_store_unreadable, published_generation_name, e);
defer store_file.close(io);
const store_size = (store_file.stat(io) catch |e|
return reportUnreadable(streams, .published_store_unreadable, published_generation_name, e)).size;
var file_storage = storage.FileStorage.init(io, store_file, store_size);
var log = logstore.LogStore.openReadOnly(gpa, file_storage.storageFile()) catch |e|
return reportUnreadable(streams, .published_store_unreadable, published_generation_name, e);
defer log.deinit(gpa);
var checkout_buf: [checkout.max_ck_payload + 1]u8 = undefined;
const state = try checkout.read(io, temporary, &checkout_buf) orelse return error.MissingCheckout;
// Imports deliberately publish an unborn checkout: the repository is
// discoverable, while the first update alone materializes its default tip.
// Whether the branch it names is present and correct is the pipeline's
// `checkout_invalid` clause, so it is not also pre-checked here -- one
// statement of the rule, and the one that reaches the operator by name.
try hooks.check(.verify_map);
var map_file = temporary.openFile(io, ".zit-import-map.jsonl", .{ .mode = .read_only }) catch |e|
return reportUnreadable(streams, .map_unreadable, ".zit-import-map.jsonl", e);
defer map_file.close(io);
const map_buf = try gpa.alloc(u8, max_map_record_bytes);
defer gpa.free(map_buf);
var map_reader = map_file.reader(io, map_buf);
var witness_file = temporary.openFile(io, witness_spill_name, .{ .mode = .read_only }) catch |e|
return reportUnreadable(streams, .witness_unreadable, witness_spill_name, e);
defer witness_file.close(io);
var witness_buf: [64 * 1024]u8 = undefined;
var witness_reader = witness_file.reader(io, &witness_buf);
const refs = temporary.readFileAlloc(io, "refs", gpa, .limited(max_verified_text_bytes)) catch |e|
return reportUnreadable(streams, .ref_target_mismatch, "refs", e);
defer gpa.free(refs);
try verifier.verify(gpa, .{
.mirror = &mirror_log,
.published = &log,
.witness = witness,
.witness_commits = &witness_reader,
.drop_names = drop_names,
.map = &map_reader,
.map_read = hooks.map_read_observer,
.refs = refs,
.refs_header = packed_refs_header,
.checkout_branch = state.branch,
.quiet = quiet,
.workers = workers,
.min_run = min_run,
.checkin_window = checkin_window,
.progress = progress,
}, streams.err);
}
/// `Repository.warmCache`, narrated unless `--quiet`.
pub fn warmCache(gpa: std.mem.Allocator, io: std.Io, request: Request, streams: cli.Streams, published: std.Io.Dir, workers: *zit.store_workers.Pool) void {
var warm: ?zit.progress.Meter = if (progressOf(request, io, streams)) |p| .{
.out = p.out,
.io = p.io,
.what = "import-git: warming cache",
.unit = "frames",
.total = 0,
.interval_ns = p.interval_ns,
.stride = p.stride,
} else null;
if (warm != null) {
streams.err.print("import-git: published, warming cache\n", .{}) catch {};
streams.err.flush() catch {};
}
zit.Repository.warmCache(gpa, io, published, workers, if (warm) |*m| m else null);
}
/// Where the phases after ingest narrate themselves, or null under `--quiet`.
fn progressOf(request: Request, io: std.Io, streams: cli.Streams) ?verifier.Progress {
if (request.quiet) return null;
return .{
.out = streams.err,
.io = io,
.interval_ns = request.progress_interval_ns,
.stride = if (request.progress_interval_ns == 0) 1 else zit.progress.Meter.default_stride,
};
}
/// Refuse the way the pipeline refuses. Opening the evidence is as much a
/// verification step as reading it, and an operator who is handed a bare
/// `FileNotFound` from inside the transaction cannot tell a vanished generation
/// from a torn header from a map that outgrew its ceiling. Same token shape,
/// same error values, so the whole surface reads as one contract.
fn reportUnreadable(
streams: cli.Streams,
clause: verifier.Clause,
what: []const u8,
cause: anyerror,
) verifier.Error {
streams.err.print("import-git: verifier: {s}: {s}: {s}\n", .{ clause.token(), what, @errorName(cause) }) catch {};
return clause.err();
}
/// The ceiling on `refs`, which the verifier reads whole. One line per ref and
/// tag, so even a history with millions of refs stays far below it.
const max_verified_text_bytes: usize = 1 << 30;
/// The longest line `git fast-export` may hand the import, and so the longest
/// identity a map record can carry.
const max_fast_export_line_bytes: usize = 1024 * 1024;
/// The longest map record the verifier admits. The map itself is streamed,
/// since it grows with the history (about 1.1 KB a commit, so past 1 GiB at a
/// million commits). A record holds at most two identities, each from one
/// fast-export line and escaped at up to six bytes a byte (`\u00XX`), so this
/// bound refuses only a corrupt map.
const max_map_record_bytes: usize = 16 * max_fast_export_line_bytes;
/// Whether a `git --version` line names a git older than `major.minor`.
/// Tolerant by design: an unparseable line answers FALSE, so an exotic build
/// string never blocks an import the real version check could not judge —
/// the flag rejection it would hit next still fails honestly. Handles the
/// common suffixed forms ("git version 2.39.3 (Apple Git-146)").
fn gitOlderThan(version_line: []const u8, major: u32, minor: u32) bool {
const prefix = "git version ";
if (!std.mem.startsWith(u8, version_line, prefix)) return false;
var it = std.mem.splitScalar(u8, version_line[prefix.len..], '.');
const got_major = std.fmt.parseUnsigned(u32, it.first(), 10) catch return false;
const minor_text = it.next() orelse return false;
// The minor field may carry a non-numeric tail in exotic builds; digits
// are read greedily and anything after them ignored.
var digits: usize = 0;
while (digits < minor_text.len and std.ascii.isDigit(minor_text[digits])) digits += 1;
if (digits == 0) return false;
const got_minor = std.fmt.parseUnsigned(u32, minor_text[0..digits], 10) catch return false;
if (got_major != major) return got_major < major;
return got_minor < minor;
}
test "squash pairing reads GitHub's ` (#N)` subjects and `refs/pull/N/head` refs, nothing looser" {
try std.testing.expectEqual(@as(?u32, 858), squashPullNumber("Fix journal key reuse (#858)"));
try std.testing.expectEqual(@as(?u32, 7), squashPullNumber("Add b (#7) "));
try std.testing.expectEqual(@as(?u32, null), squashPullNumber("Fix (#858) later"));
try std.testing.expectEqual(@as(?u32, null), squashPullNumber("Fix #858"));
try std.testing.expectEqual(@as(?u32, null), squashPullNumber("Fix (#)"));
try std.testing.expectEqual(@as(?u32, null), squashPullNumber("Fix (#12a)"));
try std.testing.expectEqual(@as(?u32, null), squashPullNumber("(#1)"));
try std.testing.expectEqual(@as(?u32, null), squashPullNumber("Fix (#99999999999)"));
try std.testing.expectEqual(@as(?u32, 858), pullHeadNumber("refs/pull/858/head"));
try std.testing.expectEqual(@as(?u32, null), pullHeadNumber("refs/pull/858/merge"));
try std.testing.expectEqual(@as(?u32, null), pullHeadNumber("refs/pull//head"));
try std.testing.expectEqual(@as(?u32, null), pullHeadNumber("refs/heads/pull/1/head"));
}
test "gitOlderThan: distro and exotic version lines judge correctly" {
try std.testing.expect(gitOlderThan("git version 2.43.0", 2, 45));
try std.testing.expect(gitOlderThan("git version 2.39.3 (Apple Git-146)", 2, 45));
try std.testing.expect(!gitOlderThan("git version 2.45.0", 2, 45));
try std.testing.expect(!gitOlderThan("git version 2.55.0", 2, 45));
try std.testing.expect(!gitOlderThan("git version 3.0", 2, 45));
// Unparseable lines never block: the answer is "not provably old".
try std.testing.expect(!gitOlderThan("gobbledygook", 2, 45));
try std.testing.expect(!gitOlderThan("git version future", 2, 45));
}
pub fn putSha1Artifact(log: *logstore.LogStore, bytes: []const u8) !void {
var digest: [std.crypto.hash.Sha1.digest_length]u8 = undefined;
std.crypto.hash.Sha1.hash(bytes, &digest, .{});
_ = try log.putArtifact(&digest, bytes, null);
}
/// Writes the commit map a later `--into` import reads: the source's identity,
/// the settings this import ran under, every commit's check-in and label, and
/// every ref's peeled tip.
fn writeMap(
gpa: std.mem.Allocator,
io: std.Io,
temporary: std.Io.Dir,
request: Request,
child: ChildFactory,
importer: *const git_import.Importer,
finalized: *const git_import.FinalizedRefs,
pinned_k: u4,
shape: editor.TreeFormat.Shape,
artifact_count: u64,
) !void {
const describe = child.vtable.describe_source orelse return error.SourceIdentityUnavailable;
var identity = try describe(child.context, gpa, request.source);
defer identity.deinit(gpa);
// A commit the stream gave no OID cannot be named to Git again, so it is
// left out, as is a ref that points at one; every real export carries
// OIDs, so this only thins a map made from a hand-written stream.
const added = importer.commitRecordCount();
var commits: std.ArrayList(zit.git_import_map.CommitRecord) = .empty;
defer commits.deinit(gpa);
var oid_of_checkin: std.StringHashMapUnmanaged([]const u8) = .empty;
defer oid_of_checkin.deinit(gpa);
for (0..added) |index| {
const record = importer.commitRecord(index) catch |err| switch (err) {
error.NoOid => continue,
else => return err,
};
try commits.append(gpa, .{ .oid = record.oid, .checkin = record.hash, .branch = record.branch, .depth = record.depth, .height = record.tree_height });
try oid_of_checkin.put(gpa, record.hash, record.oid);
}
var refs: std.ArrayList(zit.git_import_map.RefRecord) = .empty;
defer refs.deinit(gpa);
for (finalized.refs) |mapping| {
const oid = oid_of_checkin.get(mapping.target) orelse continue;
try refs.append(gpa, .{ .name = mapping.source, .oid = oid });
}
try zit.git_import_map.write(gpa, io, temporary, .{
.object_format = if (std.mem.eql(u8, identity.object_format, "sha256")) .sha256 else .sha1,
.roots = identity.roots,
.settings = .{
.submanifest = switch (shape) {
.flat => .flat,
.tree => .{ .tree = pinned_k },
},
.attribution = switch (request.attribution orelse .committer) {
.committer => .committer,
.author => .author,
},
.refuse_signed = request.refuse_signed,
.default_branch = finalized.primary_source,
},
.mirror_classifier_fired = importer.mirrorCensus().commits_with_candidates != 0,
.artifact_count = artifact_count,
.commits = commits.items,
.refs = refs.items,
});
}
/// The header every packed ref file opens with. Versioned because this is the
/// only description of the format that ships with a repository.
pub const packed_refs_header = "# zit packed-refs v1\n";
/// Writes every finalized ref into one sorted file at `refs`.
///
/// One file per ref was the obvious encoding and the wrong one: a ref's name
/// became a path, so the filesystem got a vote on which repositories are
/// representable. Two refs differing only in case are legal and distinct in
/// git — sqlite has a pair at different commits — and one silently landed on
/// the other, surfacing as a bare `PathAlreadyExists` at publication after the
/// whole import had run. Case is only the reachable instance; path length,
/// reserved device names, and each platform's own Unicode fold table are the
/// same bug wearing different clothes.
///
/// A single file has no such vote. Lines are `<canonical> <target>`, sorted by
/// canonical name, which makes the file byte-identical for a given ref set and
/// so comparable across two imports of one repository. Ref names cannot
/// contain a space or a control character, so the separator needs no escape.
/// Nothing in zit reads this back — `refs` is a reserved root the working-tree
/// scan skips, and branches resolve through repostate's `sym-` tags — so it is
/// interop output, and its whole job is to be legible.
fn writeRefs(gpa: std.mem.Allocator, io: std.Io, temporary: std.Io.Dir, refs: []const git_import.FinalizedRef) !void {
const ordered = try gpa.alloc(git_import.FinalizedRef, refs.len);
defer gpa.free(ordered);
@memcpy(ordered, refs);
std.mem.sort(git_import.FinalizedRef, ordered, {}, struct {
fn lessThan(_: void, a: git_import.FinalizedRef, b: git_import.FinalizedRef) bool {
return std.mem.lessThan(u8, a.canonical, b.canonical);
}
}.lessThan);
var body: std.ArrayList(u8) = .empty;
defer body.deinit(gpa);
try body.appendSlice(gpa, packed_refs_header);
for (ordered) |mapping| {
try body.appendSlice(gpa, mapping.canonical);
try body.append(gpa, ' ');
try body.appendSlice(gpa, mapping.target);
try body.append(gpa, '\n');
}
var file = try temporary.createFile(io, "refs", .{ .read = true, .exclusive = true });
defer file.close(io);
try file.writeStreamingAll(io, body.items);
try file.sync(io);
}
fn rawHashName(name: []const u8, buffer: *[32]u8) ?[]const u8 {
if (name.len != 40 and name.len != 64) return null;
const raw = buffer[0 .. name.len / 2];
_ = std.fmt.hexToBytes(raw, name) catch return null;
return raw;
}
pub fn finishSession(
pipes: *ExportPipes,
streams: cli.Streams,
child: ChildFactory,
quiet: bool,
abort: bool,
) !bool {
var lifecycle_error: ?anyerror = null;
// EOF is observed by the importing thread before it starts cancellation.
// A child that is a zombie can still accept `kill`, so this observation is
// the race-free completion evidence; `reap` below supplies its actual code.
var child_finished_before_cancellation = abort and if (pipes.owner) |owner| owner.stdout_reached_eof else false;
if (abort) {
child.vtable.terminate(child.context) catch |err| {
if (err == error.ChildAlreadyExited) {
child_finished_before_cancellation = true;
} else {
lifecycle_error = err;
}
};
}
child.vtable.drain_stderr(child.context) catch |err| {
if (lifecycle_error == null) lifecycle_error = err;
};
child.vtable.reap(child.context) catch |err| {
if (lifecycle_error == null) lifecycle_error = err;
};
const enabled = abort or terminalExitCode(pipes) != 0 or !quiet;
if (pipes.diagnostics) |write_diagnostics| {
write_diagnostics(pipes, streams.err, enabled);
} else {
copyDiagnostics(&pipes.stderr, streams.err, enabled);
}
if (lifecycle_error) |err| return err;
return child_finished_before_cancellation;
}
pub fn terminalExitCode(pipes: *const ExportPipes) u8 {
return if (pipes.exit_code_ref) |code| code.* else pipes.exit_code;
}
fn fileExists(io: std.Io, dir: std.Io.Dir, name: []const u8) bool {
var file = dir.openFile(io, name, .{}) catch return false;
file.close(io);
return true;
}
fn destinationExists(io: std.Io, parent: std.Io.Dir, name: []const u8) bool {
var directory = parent.openDir(io, name, .{}) catch {
var file = parent.openFile(io, name, .{}) catch return false;
file.close(io);
return true;
};
directory.close(io);
return true;
}
fn copyDiagnostics(reader: *std.Io.Reader, err: *std.Io.Writer, enabled: bool) void {
if (enabled) _ = reader.streamRemaining(err) catch {} else _ = reader.discard(.unlimited) catch {};
}
pub const CommandScript = struct {
pub const Preflight = enum { branch_head, detached_head, git_missing, not_repository, missing_objects, shallow, partial_promisor };
/// The stream `cancel_export_mid_stream` substitutes for the configured export.
///
/// Named rather than inlined because a test asserts what this stream DOES:
/// one blob parses, then a record the reader rejects -- a truncation, not a
/// rejection at record zero, so the published store is already open when
/// the cancellation lands. A test that hand-copied these bytes would keep
/// passing against a stale copy the moment this literal changed, and its
/// claim about the window would quietly stop being true.
pub const cancelled_export = "feature done\nblob\nmark :1\ndata 1\nx\ninvalid\n";
pub const StagingReplacement = enum { directory, symlink };
/// One deliberate divergence between the frozen stores and what the
/// import decided, injected at `.verify_mirror_store` -- after the
/// published store and the mirror store are closed files and before the
/// verifier reads either.
///
/// This is the CORRUPTION seam, distinct from `PublicationHooks.fail_at`,
/// which is the FAILURE seam. A failure proves the transaction unwinds; a
/// corruption proves the verifier looks. Before this existed the two
/// mutating seams (`replaceStaging`, `plantStrayStagingFile`) both fired
/// after `advance(.verified)`, so nothing in the suite could reach the
/// bytes the verifier is supposed to be reading.
///
/// Each member is named for the invariant it must trip, NOT for the
/// mechanism it uses. A fixture that fails at some other clause has not
/// proven its clause exists.
pub const VerifierFault = enum {
/// An artifact the classifier authorized dropping is present in the
/// published store.
retained_omitted_artifact,
/// An artifact the import wrote is absent from the published store.
dropped_required_artifact,
/// A well-formed artifact the import never wrote is present in the
/// published store.
added_artifact,
/// One artifact the import wrote replaced by one it never wrote. The
/// artifact COUNT is unchanged, so only the name fingerprint can see
/// it.
substituted_artifact,
/// A name the import wrote, republished over bytes that do not hash
/// to it. The name set is untouched; only the content moved.
changed_artifact_bytes,
/// Two artifacts' bytes changed, the first and the last blob. Only
/// the earlier in persisted order may be reported, however the
/// verifier's sweep divides the store between threads.
changed_two_artifacts_bytes,
/// An artifact of the drop set, in the mirror store, over bytes that
/// do not hash to its name. The published store is untouched: the
/// drop set is content nothing but the verifier reads.
changed_dropped_artifact_bytes,
/// A delta frame whose recorded base is not the base its delta was
/// encoded against. Every name is right and every frame checksums;
/// only reconstruction through the chain reveals it.
wrong_delta_base,
/// The check-in's F cards permuted onto different paths. Every blob
/// still present, every artifact still hashes -- the flattened tree is
/// the only thing that moved.
tree_change_preserving_artifacts,
/// A merge check-in's P card with its parents in the other order.
parent_reorder,
/// A ref row in `refs` pointed at a different check-in.
ref_retarget,
/// A tag row in `refs` retargeted, and its kind flipped between the
/// lightweight and annotated spellings.
tag_change,
/// The finalized JSONL's author and committer identities exchanged, so
/// the record disagrees with the U card the store actually carries.
attribution_change,
/// One hex digit of a `provenance_digest` value in the finalized JSONL
/// flipped. Nothing else in the import moves.
provenance_tamper,
/// One record appended to the finalized JSONL, longer than the
/// verifier admits.
oversized_map_record,
/// Not a corruption: the finalized JSONL grown past the verifier's
/// record buffer with records of a kind it skips. The map is read a
/// record at a time, so the import still publishes.
map_past_one_buffer,
/// One byte of the spilled witness flipped, in the middle of the
/// file. Store and map are untouched; only the record of what the
/// import decided moved.
changed_witness_byte,
/// The spilled witness short by its last byte.
truncated_witness,
/// One hex digit of the last commit record's `zit` flipped in place,
/// same length, once the verifier has read the first window of map
/// records again -- while the rest may sit in its read buffer. No
/// clause may judge a map other than the one it indexed.
map_rewritten_after_read,
};
pub const Config = struct {
preflight: Preflight = .branch_head,
fast_export: []const u8 = "",
stderr_bytes: usize = 0,
diagnostic_ring_limit: usize = 4096,
/// Substitute `cancelled_export` for `fast_export`: a stream that parses
/// one blob and then hits a record the reader rejects.
///
/// A BOOLEAN, not a count. It was declared as `?usize` and only ever
/// tested for null, so the number was dead and the name promised a
/// cancellation after N events that no code implements -- the mechanism
/// is a stream substitution and nothing counts anything.
cancel_export_mid_stream: bool = false,
competing_destination_at_publish: bool = false,
export_exit: u8 = 0,
replace_staging_at_publish: ?StagingReplacement = null,
replace_destination_after_rename: bool = false,
/// Drop an unexpected file into staging after the mirror store has
/// been discarded -- i.e. between the last step that removes anything
/// and the rename that ships whatever is left. Stands in for a future
/// edit that writes an intermediate file and forgets to clean it up.
stray_staging_file_after_discard: bool = false,
/// Corrupt staging between the freeze and the verifier's first read.
verifier_fault: ?VerifierFault = null,
/// Corrupt the witness after it is recorded.
witness_fault: ?WitnessFault = null,
limits: ImportLimits = .{},
};
pub const Publication = struct {
store_reopened_for_verify: bool = false,
map_reopened_for_verify: bool = false,
verify_before_publish: bool = false,
/// Stage ordering, each recorded as "was the PREVIOUS stage already
/// complete when this one began?" rather than as a bare visit count
/// -- a visit count would still be set by a body that ran the stages
/// in the wrong order.
store_frozen: bool = false,
mirror_store_created: bool = false,
mirror_store_frozen: bool = false,
frozen_before_verify: bool = false,
both_stores_present_before_discard: bool = false,
mirror_store_discarded: bool = false,
discard_before_rename: bool = false,
};
pub const Environment = struct {
pub fn get(_: Environment, name: []const u8) ?[]const u8 {
if (std.mem.eql(u8, name, "GIT_TERMINAL_PROMPT")) return "0";
if (std.mem.eql(u8, name, "GIT_NO_LAZY_FETCH")) return "1";
return null;
}
};
preflight: Preflight,
fast_export: []const u8,
stderr_bytes: usize,
diagnostic_ring_limit: usize,
cancel_export_mid_stream: bool,
competing_destination_at_publish: bool,
export_exit: u8,
replace_staging_at_publish: ?StagingReplacement,
replace_destination_after_rename: bool,
stray_staging_file_after_discard: bool,
verifier_fault: ?VerifierFault,
witness_fault: ?WitnessFault,
/// Whether the injection actually happened. A corruption fixture that
/// asserts a verifier error without asserting this would pass on a run
/// where the injection silently did nothing -- which is the one failure
/// mode a corruption suite must not have.
verifier_fault_injected: bool = false,
/// The observer the caller handed to `dependencies`, chained behind the
/// script's own rather than overwritten by it.
caller_observer: ?PublicationHooks.Observer = null,
limits: ImportLimits,
default_branch: ?[]const u8 = null,
detached_head_recorded: bool = false,
export_starts: usize = 0,
/// How many known tips the last export start was given, or null when it
/// was a fresh import's start.
known_tips_seen: ?usize = null,
/// What `describe_source` reports as the source's parentless commits.
source_roots: []const []const u8 = &.{},
/// What `describe_source` reports as the refs the source publishes now.
source_refs: []const SourceRef = &.{},
preflight_starts: usize = 0,
lifecycle: ChildLifecycle = .{},
stderr_drained_bytes: usize = 0,
environment: Environment = .{},
publication: Publication = .{},
competitor_destination_intact: bool = false,
temporary_sibling_left: bool = false,
staging_replacement_observed: bool = false,
staging_replacement_name: [64]u8 = undefined,
staging_replacement_name_len: usize = 0,
destination_parent: ?std.Io.Dir = null,
destination_name: []const u8 = "",
io: ?std.Io = null,
pub fn init(config: Config) CommandScript {
return .{
.preflight = config.preflight,
.fast_export = config.fast_export,
.stderr_bytes = config.stderr_bytes,
.diagnostic_ring_limit = config.diagnostic_ring_limit,
.cancel_export_mid_stream = config.cancel_export_mid_stream,
.competing_destination_at_publish = config.competing_destination_at_publish,
.export_exit = config.export_exit,
.replace_staging_at_publish = config.replace_staging_at_publish,
.replace_destination_after_rename = config.replace_destination_after_rename,
.stray_staging_file_after_discard = config.stray_staging_file_after_discard,
.verifier_fault = config.verifier_fault,
.witness_fault = config.witness_fault,
.limits = config.limits,
};
}
fn dependencies(self: *CommandScript, fault: PublicationHooks) Dependencies {
var publication = fault;
// The script needs its own notification to drive `publicationEvent`,
// but a caller's observer is a QUESTION the caller asked, and dropping
// it answers that question with silence: every assertion made through
// a discarded observer passes vacuously and nothing says why. Chained
// rather than replaced, so both are served.
self.caller_observer = fault.observer;
publication.observer = .{ .context = self, .notify = publicationEvent };
publication.staging_observer = .{ .context = self, .replace = replaceStaging };
publication.map_read_observer = .{ .context = self, .read = mapRead };
return .{
.git = .{ .context = self, .vtable = &.{
.start_export = startExport,
.describe_source = describeSource,
.terminate = terminate,
.drain_stderr = drainStderr,
.reap = reap,
} },
.publication = publication,
.limits = self.limits,
.witness_fault = self.witness_fault,
};
}
fn describeSource(context: *anyopaque, gpa: std.mem.Allocator, _: []const u8) !SourceIdentity {
const self: *CommandScript = @ptrCast(@alignCast(context));
const object_format = try gpa.dupe(u8, "sha1");
errdefer gpa.free(object_format);
const roots = try gpa.alloc([]u8, self.source_roots.len);
errdefer gpa.free(roots);
var built: usize = 0;
errdefer for (roots[0..built]) |root| gpa.free(root);
for (self.source_roots, roots) |root, *slot| {
slot.* = try gpa.dupe(u8, root);
built += 1;
}
const refs = try gpa.alloc(SourceRef, self.source_refs.len);
errdefer gpa.free(refs);
var refs_built: usize = 0;
errdefer for (refs[0..refs_built]) |ref| {
gpa.free(ref.name);
gpa.free(ref.oid);
};
for (self.source_refs, refs) |ref, *slot| {
const name = try gpa.dupe(u8, ref.name);
errdefer gpa.free(name);
slot.* = .{ .name = name, .oid = try gpa.dupe(u8, ref.oid) };
refs_built += 1;
}
return .{ .object_format = object_format, .roots = roots, .refs = refs };
}
fn startExport(context: *anyopaque, _: []const u8, known: ?KnownHistory) !ExportPipes {
const self: *CommandScript = @ptrCast(@alignCast(context));
self.export_starts += 1;
self.known_tips_seen = if (known) |k| k.tips.len else null;
self.lifecycle.launched += 1;
const export_bytes = if (self.cancel_export_mid_stream)
cancelled_export
else
self.fast_export;
return .{ .stdout = .fixed(export_bytes), .stderr = .fixed("scripted diagnostic\n"), .exit_code = self.export_exit };
}
fn terminate(context: *anyopaque) !void {
const self: *CommandScript = @ptrCast(@alignCast(context));
self.lifecycle.terminated += 1;
}
fn drainStderr(context: *anyopaque) !void {
const self: *CommandScript = @ptrCast(@alignCast(context));
self.lifecycle.stderr_drained += 1;
self.stderr_drained_bytes = self.stderr_bytes;
}
fn reap(context: *anyopaque) !void {
const self: *CommandScript = @ptrCast(@alignCast(context));
self.lifecycle.reaped += 1;
}
fn publicationEvent(context: *anyopaque, point: PublicationHooks.FailurePoint) void {
const self: *CommandScript = @ptrCast(@alignCast(context));
switch (point) {
.after_close_store => self.publication.store_frozen = true,
// Asked of the filesystem, not asserted. `create_mirror_store`
// fires BEFORE the create, so a later point is the first at which
// the question has an answer -- and an unconditional `true` here
// would be exactly the vacuous observation this suite already got
// wrong once with `temporary_sibling_left`.
.copy_mirror_survivors => self.publication.mirror_store_created = self.stagingHolds(mirror_store_name),
.after_close_mirror_store => self.publication.mirror_store_frozen = true,
// The one instant at which both stores are closed files and the
// verifier has read neither.
.verify_mirror_store => if (self.verifier_fault) |fault| self.injectVerifierFault(fault),
.verify_store => {
self.publication.store_reopened_for_verify = true;
self.publication.frozen_before_verify = self.publication.store_frozen and self.publication.mirror_store_frozen;
},
.verify_map => self.publication.map_reopened_for_verify = true,
.before_discard_mirror_store => self.publication.both_stores_present_before_discard = self.stagingHoldsBothStores(),
.after_discard_mirror_store => {
self.publication.mirror_store_discarded = !self.stagingHolds(mirror_store_name);
if (self.stray_staging_file_after_discard) self.plantStrayStagingFile();
},
.before_rename_no_replace => {
self.publication.discard_before_rename = self.publication.mirror_store_discarded;
self.publication.verify_before_publish = self.publication.store_reopened_for_verify and self.publication.map_reopened_for_verify;
if (self.competing_destination_at_publish) {
const parent = self.destination_parent orelse return;
parent.createDir(self.io orelse return, self.destination_name, .default_dir) catch return;
self.competitor_destination_intact = true;
}
},
.after_rename_no_replace => if (self.replace_destination_after_rename) {
const parent = self.destination_parent orelse return;
const io = self.io orelse return;
parent.deleteTree(io, self.destination_name) catch return;
parent.createDir(io, self.destination_name, .default_dir) catch return;
var replacement = parent.openDir(io, self.destination_name, .{ .follow_symlinks = false }) catch return;
defer replacement.close(io);
var marker = replacement.createFile(io, "replacement-sentinel", .{ .read = true, .exclusive = true }) catch return;
defer marker.close(io);
marker.writeStreamingAll(io, "survives\n") catch return;
self.competitor_destination_intact = true;
},
else => {},
}
// After the script's own bookkeeping: an observer that sees a boundary
// sees it in the state the script left it in.
if (self.caller_observer) |observer| observer.notify(observer.context, point);
}
/// Whether the staging sibling holds BOTH store files right now. This is
/// the direct evidence that the two stores are distinct files: it is
/// asked at the one instant both are supposed to exist, from outside the
/// transaction, by name.
fn stagingHoldsBothStores(self: *CommandScript) bool {
return self.stagingHolds(mirror_store_name) and
self.stagingHolds(published_generation_name);
}
/// Write a file into staging that the publish manifest does not name.
fn plantStrayStagingFile(self: *CommandScript) void {
const parent = self.destination_parent orelse return;
const io = self.io orelse return;
var it_dir = parent.openDir(io, ".", .{ .iterate = true }) catch return;
defer it_dir.close(io);
var entries = it_dir.iterate();
while (entries.next(io) catch return) |entry| {
if (!std.mem.startsWith(u8, entry.name, ".zit-tmp-import-")) continue;
var staging = parent.openDir(io, entry.name, .{ .follow_symlinks = false }) catch return;
defer staging.close(io);
var stray = staging.createFile(io, "unpublishable-scratch", .{ .read = true, .exclusive = true }) catch return;
defer stray.close(io);
stray.writeStreamingAll(io, "scratch\n") catch return;
return;
}
}
// ---- Corruption seam -------------------------------------------------
//
// Everything below runs at `.verify_mirror_store`, from OUTSIDE the
// transaction, against staging as a plain directory. It never touches the
// production path; a run with `verifier_fault == null` executes none of it.
/// Open the staging sibling. Caller closes.
fn openStagingDir(self: *CommandScript) ?std.Io.Dir {
const parent = self.destination_parent orelse return null;
const io = self.io orelse return null;
var it_dir = parent.openDir(io, ".", .{ .iterate = true }) catch return null;
defer it_dir.close(io);
var entries = it_dir.iterate();
while (entries.next(io) catch return null) |entry| {
if (!std.mem.startsWith(u8, entry.name, ".zit-tmp-import-")) continue;
return parent.openDir(io, entry.name, .{ .follow_symlinks = false }) catch return null;
}
return null;
}
fn injectVerifierFault(self: *CommandScript, fault: VerifierFault) void {
const io = self.io orelse return;
var staging = self.openStagingDir() orelse return;
defer staging.close(io);
switch (fault) {
.ref_retarget, .tag_change => self.rewriteStagingRefs(staging, fault),
.attribution_change, .provenance_tamper, .oversized_map_record, .map_past_one_buffer => self.rewriteStagingProvenance(staging, fault),
.changed_dropped_artifact_bytes => self.rewriteMirrorDropped(staging),
.changed_witness_byte, .truncated_witness => self.rewriteWitness(staging, fault),
// Injected at `mapRead`, once the verifier has read the map.
.map_rewritten_after_read => {},
else => self.rewritePublishedStore(staging, fault),
}
}
fn rewriteWitness(self: *CommandScript, staging: std.Io.Dir, fault: VerifierFault) void {
const body = self.readStagingText(staging, witness_spill_name) orelse return;
defer std.heap.page_allocator.free(body);
if (body.len < 2) return;
switch (fault) {
.changed_witness_byte => {
body[body.len / 2] ^= 0x01;
self.writeStagingText(staging, witness_spill_name, body);
},
else => self.writeStagingText(staging, witness_spill_name, body[0 .. body.len - 1]),
}
}
fn mapRead(context: *anyopaque) void {
const self: *CommandScript = @ptrCast(@alignCast(context));
if (self.verifier_fault != .map_rewritten_after_read or self.verifier_fault_injected) return;
const io = self.io orelse return;
var staging = self.openStagingDir() orelse return;
defer staging.close(io);
const body = self.readStagingText(staging, ".zit-import-map.jsonl") orelse return;
defer std.heap.page_allocator.free(body);
const marker = "\"kind\":\"commit\",\"oid\":";
const commit = std.mem.lastIndexOf(u8, body, marker) orelse return;
const zit_marker = "\"zit\":\"";
const at = (std.mem.indexOfPos(u8, body, commit, zit_marker) orelse return) + zit_marker.len;
if (!flipHexDigit(&body[at])) return;
// In place: the verifier holds the file open.
var file = staging.openFile(io, ".zit-import-map.jsonl", .{ .mode = .read_write }) catch return;
defer file.close(io);
file.writePositionalAll(io, body[at..][0..1], at) catch return;
self.verifier_fault_injected = true;
}
fn readStagingText(self: *CommandScript, staging: std.Io.Dir, name: []const u8) ?[]u8 {
const io = self.io orelse return null;
return staging.readFileAlloc(io, name, std.heap.page_allocator, .limited(1 << 20)) catch null;
}
fn writeStagingText(self: *CommandScript, staging: std.Io.Dir, name: []const u8, bytes: []const u8) void {
const io = self.io orelse return;
staging.deleteFile(io, name) catch return;
var file = staging.createFile(io, name, .{ .read = true, .exclusive = true }) catch return;
defer file.close(io);
file.writeStreamingAll(io, bytes) catch return;
file.sync(io) catch return;
self.verifier_fault_injected = true;
}
/// Flip one hex digit in place. The result is still a syntactically valid
/// name, so nothing upstream of the clause under test can reject it on
/// shape alone.
fn flipHexDigit(byte: *u8) bool {
byte.* = switch (byte.*) {
'0' => '1',
'1' => '0',
'a'...'e' => byte.* + 1,
'f' => 'a',
'2'...'9' => byte.* - 1,
else => return false,
};
return true;
}
fn rewriteStagingRefs(self: *CommandScript, staging: std.Io.Dir, fault: VerifierFault) void {
const body = self.readStagingText(staging, "refs") orelse return;
defer std.heap.page_allocator.free(body);
const want: []const u8 = if (fault == .tag_change) "sym-tags/" else "sym-heads/";
var cursor: usize = 0;
while (std.mem.indexOfPos(u8, body, cursor, want)) |start| {
const end = std.mem.indexOfScalarPos(u8, body, start, '\n') orelse body.len;
const space = std.mem.lastIndexOfScalar(u8, body[start..end], ' ') orelse {
cursor = end;
continue;
};
// Retarget: the row now names something that is not the check-in
// the import resolved for it.
if (!flipHexDigit(&body[start + space + 1])) {
cursor = end;
continue;
}
// Type change, for the tag case only: the same row reappears
// spelled as a branch, which is a different kind of ref entirely.
// The two prefixes differ in length, so this rebuilds the file
// rather than poking it -- every other byte is carried through.
if (fault == .tag_change) {
const gpa = std.heap.page_allocator;
var rebuilt: std.ArrayList(u8) = .empty;
defer rebuilt.deinit(gpa);
rebuilt.appendSlice(gpa, body[0..start]) catch return;
rebuilt.appendSlice(gpa, "sym-heads/") catch return;
rebuilt.appendSlice(gpa, body[start + want.len ..]) catch return;
self.writeStagingText(staging, "refs", rebuilt.items);
return;
}
self.writeStagingText(staging, "refs", body);
return;
}
}
fn rewriteStagingProvenance(self: *CommandScript, staging: std.Io.Dir, fault: VerifierFault) void {
const gpa = std.heap.page_allocator;
const body = self.readStagingText(staging, ".zit-import-map.jsonl") orelse return;
defer gpa.free(body);
switch (fault) {
.provenance_tamper => {
const marker = "\"provenance_digest\":\"";
const at = std.mem.indexOf(u8, body, marker) orelse return;
if (!flipHexDigit(&body[at + marker.len])) return;
self.writeStagingText(staging, ".zit-import-map.jsonl", body);
},
.attribution_change => {
// Exchange the two identity objects on the first commit line.
// Attribution mode selects ONE of them; swapping makes the
// record name the identity the store's U card does not carry,
// while every digit of both identities stays well-formed.
const author_marker = "\"author\":";
const committer_marker = ",\"committer\":";
const tree_marker = ",\"tree_digest\":";
const a = std.mem.indexOf(u8, body, author_marker) orelse return;
const c = std.mem.indexOfPos(u8, body, a, committer_marker) orelse return;
const t = std.mem.indexOfPos(u8, body, c, tree_marker) orelse return;
const author = body[a + author_marker.len .. c];
const committer = body[c + committer_marker.len .. t];
if (std.mem.eql(u8, author, committer)) return;
var swapped: std.ArrayList(u8) = .empty;
defer swapped.deinit(gpa);
swapped.appendSlice(gpa, body[0 .. a + author_marker.len]) catch return;
swapped.appendSlice(gpa, committer) catch return;
swapped.appendSlice(gpa, committer_marker) catch return;
swapped.appendSlice(gpa, author) catch return;
swapped.appendSlice(gpa, body[t..]) catch return;
self.writeStagingText(staging, ".zit-import-map.jsonl", swapped.items);
},
.oversized_map_record, .map_past_one_buffer => {
const open = "{\"kind\":\"pad\",\"pad\":\"";
const close = "\"}\n";
var grown: std.ArrayList(u8) = .empty;
defer grown.deinit(gpa);
grown.appendSlice(gpa, body) catch return;
if (fault == .oversized_map_record) {
grown.appendSlice(gpa, open) catch return;
grown.appendNTimes(gpa, 'x', max_map_record_bytes) catch return;
grown.appendSlice(gpa, close) catch return;
} else {
while (grown.items.len <= max_map_record_bytes) {
grown.appendSlice(gpa, open) catch return;
grown.appendNTimes(gpa, 'x', 4096) catch return;
grown.appendSlice(gpa, close) catch return;
}
}
self.writeStagingText(staging, ".zit-import-map.jsonl", grown.items);
},
else => {},
}
}
const StagedArtifact = struct { name: []u8, bytes: []u8 };
fn readGeneration(
self: *CommandScript,
staging: std.Io.Dir,
name: []const u8,
config: ?*?logstore.OwnedConfig,
hash_kind: ?*u8,
) ?[]StagedArtifact {
const io = self.io orelse return null;
const gpa = std.heap.page_allocator;
var file = staging.openFile(io, name, .{ .mode = .read_only }) catch return null;
defer file.close(io);
const size = (file.stat(io) catch return null).size;
var file_storage = storage.FileStorage.init(io, file, size);
var log = logstore.LogStore.openReadOnly(gpa, file_storage.storageFile()) catch return null;
defer log.deinit(gpa);
// Carried through the rebuild when the caller asks for it, so a
// replacement generation differs from the original in the ONE property
// its fault is named for and not also in its settings. The hash kind
// most of all: hardcoding sha1 would make every store fault silently
// inexpressible under any other algorithm -- the rebuilt header would
// disagree with its own names and the whole suite would go red with no
// diagnostic.
if (config) |out| out.* = log.readConfig(gpa) catch null;
if (hash_kind) |out| out.* = log.hash_kind;
const rids = log.ridsInPersistedOrder(gpa) catch return null;
defer gpa.free(rids);
var out: std.ArrayList(StagedArtifact) = .empty;
for (rids) |rid| {
const bytes = (log.contentUnmasked(gpa, rid, null) catch continue) orelse continue;
const hash = gpa.dupe(u8, log.hashOf(rid)) catch continue;
out.append(gpa, .{ .name = hash, .bytes = bytes }) catch continue;
}
return out.toOwnedSlice(gpa) catch null;
}
fn freeGeneration(items: []StagedArtifact) void {
const gpa = std.heap.page_allocator;
for (items) |item| {
gpa.free(item.name);
gpa.free(item.bytes);
}
gpa.free(items);
}
/// Rewrite the published store from its own contents, applying `fault`.
///
/// Rebuilding rather than byte-poking is deliberate: every frame the
/// replacement contains checksums correctly and every name is well-formed,
/// so the only thing a verifier can object to is the property the fault is
/// named for.
fn rewritePublishedStore(self: *CommandScript, staging: std.Io.Dir, fault: VerifierFault) void {
const io = self.io orelse return;
const gpa = std.heap.page_allocator;
var config: ?logstore.OwnedConfig = null;
defer if (config) |*owned| owned.deinit(gpa);
var hash_kind: u8 = 0;
const items = self.readGeneration(staging, published_generation_name, &config, &hash_kind) orelse return;
defer freeGeneration(items);
if (items.len == 0) return;
// `retained_omitted_artifact` needs a name the copy WITHHELD, which
// exists only as a mirror-store artifact the published store lacks.
var readd: ?StagedArtifact = null;
const mirror_items: ?[]StagedArtifact = if (fault == .retained_omitted_artifact)
self.readGeneration(staging, mirror_store_name, null, null)
else
null;
defer if (mirror_items) |mirror| freeGeneration(mirror);
if (mirror_items) |mirror| {
outer: for (mirror) |candidate| {
for (items) |kept| {
if (std.mem.eql(u8, kept.name, candidate.name)) continue :outer;
}
readd = candidate;
break;
}
// No drop set means nothing was omitted, so this fault has no
// referent. Leaving `verifier_fault_injected` false is the whole
// point: the fixture asserts it and fails loudly rather than
// passing on a corruption that never happened.
if (readd == null) return;
}
staging.deleteFile(io, published_generation_name) catch return;
var file = staging.createFile(io, published_generation_name, .{ .read = true, .exclusive = true }) catch return;
var file_open = true;
defer if (file_open) file.close(io);
var file_storage = storage.FileStorage.init(io, file, 0);
var log = logstore.LogStore.open(gpa, file_storage.storageFile(), hash_kind) catch return;
var log_live = true;
defer if (log_live) log.deinit(gpa);
if (config) |owned| log.putConfig(owned.config) catch return;
// When a check-in is re-emitted its name changes; every file that
// names it must follow.
const Renamed = struct { old: [2 * logstore.max_hash_len]u8, new: [2 * logstore.max_hash_len]u8, len: usize };
var renamed: ?Renamed = null;
// A BLOB, chosen by kind rather than by position. Both faults below
// want an ordinary artifact: aim either at a check-in and the fixture
// starts proving `checkin-unresolvable` instead of the clause it is
// named for -- loudly rather than falsely, but for the wrong reason.
const target = lastBlob(items) orelse return;
const skip: usize = if (fault == .dropped_required_artifact or fault == .substituted_artifact) target else std.math.maxInt(usize);
const mutate: usize = switch (fault) {
.changed_artifact_bytes, .wrong_delta_base, .changed_two_artifacts_bytes => target,
else => std.math.maxInt(usize),
};
const also_mutate: usize = if (fault == .changed_two_artifacts_bytes)
(firstBlob(items) orelse return)
else
std.math.maxInt(usize);
if (fault == .changed_two_artifacts_bytes and also_mutate == mutate) return;
for (items, 0..) |item, index| {
if (index == skip) {
self.verifier_fault_injected = true;
continue;
}
if ((index == mutate or index == also_mutate) and
(fault == .changed_artifact_bytes or fault == .changed_two_artifacts_bytes))
{
// Same name, different bytes. Nothing about the frame is
// malformed; only `hash(content) == name` is false.
_ = log.putArtifact(item.name, "verifier fixture: these are not the bytes\n", null) catch return;
self.verifier_fault_injected = true;
continue;
}
if (index == mutate and fault == .wrong_delta_base and index >= 2) {
// A delta encoded against its NEIGHBOUR -- which is similar
// enough to it to produce real copy segments -- but recorded
// against rid 1, a much shorter artifact. Both frames exist,
// both checksum, and the chain is within bounds; only walking
// it lands anywhere wrong.
//
// The neighbour matters. A delta between two dissimilar
// artifacts carries no copy segment at all, and such a delta
// reconstructs the SAME bytes from any base whatsoever -- so
// recording the wrong one would be an injection that changed
// nothing, and this fixture would then be asserting a clause
// about a store that is not actually wrong.
if (self.plantWrongDeltaBase(&log, items[index - 1].bytes, item)) continue;
}
if (fault == .tree_change_preserving_artifacts or fault == .parent_reorder) {
if (renamed == null) if (self.mutatedCheckin(item, fault)) |replacement| {
defer gpa.free(replacement.name);
defer gpa.free(replacement.bytes);
_ = log.putArtifact(replacement.name, replacement.bytes, null) catch return;
// Re-emitting a check-in necessarily renames it. Carrying
// the rename into `refs` and the map is what keeps these
// two fixtures SURGICAL: without it the ref rows and the
// provenance lines would also dangle, and the fixture
// would trip whichever clause happened to run first
// instead of the one it is named for.
var rename: Renamed = .{ .old = undefined, .new = undefined, .len = 2 * item.name.len };
_ = std.fmt.bufPrint(rename.old[0..rename.len], "{x}", .{item.name}) catch return;
_ = std.fmt.bufPrint(rename.new[0..rename.len], "{x}", .{replacement.name}) catch return;
renamed = rename;
self.verifier_fault_injected = true;
continue;
};
}
_ = log.putArtifact(item.name, item.bytes, null) catch return;
}
if (readd) |dropped| {
_ = log.putArtifact(dropped.name, dropped.bytes, null) catch return;
self.verifier_fault_injected = true;
}
if (fault == .added_artifact or fault == .substituted_artifact) {
const novel = "verifier fixture: an artifact no import produced\n";
var digest: [std.crypto.hash.Sha1.digest_length]u8 = undefined;
std.crypto.hash.Sha1.hash(novel, &digest, .{});
_ = log.putArtifact(&digest, novel, null) catch return;
self.verifier_fault_injected = true;
}
log.commit() catch return;
log.deinit(gpa);
log_live = false;
file.sync(io) catch {};
file.close(io);
file_open = false;
if (renamed) |rename| {
self.substituteStagingName(staging, "refs", rename.old[0..rename.len], rename.new[0..rename.len]);
self.substituteStagingName(staging, ".zit-import-map.jsonl", rename.old[0..rename.len], rename.new[0..rename.len]);
}
}
/// Rewrite the mirror store from its own contents with one drop-set
/// artifact's bytes replaced. Rebuilt for `rewritePublishedStore`'s
/// reason: every frame checksums and every name is the one it was, so the
/// only thing left to object to is `hash(content) == name`.
fn rewriteMirrorDropped(self: *CommandScript, staging: std.Io.Dir) void {
const io = self.io orelse return;
const gpa = std.heap.page_allocator;
var config: ?logstore.OwnedConfig = null;
defer if (config) |*owned| owned.deinit(gpa);
var hash_kind: u8 = 0;
const mirror = self.readGeneration(staging, mirror_store_name, &config, &hash_kind) orelse return;
defer freeGeneration(mirror);
const kept = self.readGeneration(staging, published_generation_name, null, null) orelse return;
defer freeGeneration(kept);
// The drop set is the mirror store's artifacts the published store
// lacks. Empty, and the fault has no referent; the flag stays false
// and the fixture says so.
const target = outer: for (mirror, 0..) |candidate, index| {
for (kept) |item| {
if (std.mem.eql(u8, item.name, candidate.name)) continue :outer;
}
break :outer index;
} else return;
staging.deleteFile(io, mirror_store_name) catch return;
var file = staging.createFile(io, mirror_store_name, .{ .read = true, .exclusive = true }) catch return;
defer file.close(io);
var file_storage = storage.FileStorage.init(io, file, 0);
var log = logstore.LogStore.open(gpa, file_storage.storageFile(), hash_kind) catch return;
defer log.deinit(gpa);
if (config) |owned| log.putConfig(owned.config) catch return;
for (mirror, 0..) |item, index| {
const bytes: []const u8 = if (index == target) "verifier fixture: not the dropped bytes\n" else item.bytes;
_ = log.putArtifact(item.name, bytes, null) catch return;
}
log.commit() catch return;
file.sync(io) catch return;
self.verifier_fault_injected = true;
}
/// The last artifact in persisted order whose bytes are NOT a check-in
/// manifest, or null when a generation is all check-ins.
fn firstBlob(items: []const StagedArtifact) ?usize {
const gpa = std.heap.page_allocator;
for (items, 0..) |item, index| {
var manifest = artifact.parse(gpa, item.bytes) catch return index;
defer manifest.deinit();
if (manifest.kind != .manifest) return index;
}
return null;
}
fn lastBlob(items: []const StagedArtifact) ?usize {
const gpa = std.heap.page_allocator;
var index = items.len;
while (index > 0) {
index -= 1;
var manifest = artifact.parse(gpa, items[index].bytes) catch return index;
defer manifest.deinit();
if (manifest.kind != .manifest) return index;
}
return null;
}
/// Store `item` as a delta of `base` under a recorded base of rid 1.
///
/// Returns whether the divergence is real, PROVEN rather than assumed:
/// applying the encoded delta to the artifact the frame actually names must
/// fail. Nothing is written and nothing is claimed when it does not, so the
/// fixture reports "the injection did not happen" instead of asserting a
/// verifier clause about a store that reconstructs perfectly.
fn plantWrongDeltaBase(
self: *CommandScript,
log: *logstore.LogStore,
base: []const u8,
item: StagedArtifact,
) bool {
const gpa = std.heap.page_allocator;
const encoded = delta.create(gpa, base, item.bytes) catch return false;
defer gpa.free(encoded);
const recorded_base = (log.contentUnmasked(gpa, 1, null) catch return false) orelse return false;
defer gpa.free(recorded_base);
if (delta.apply(gpa, recorded_base, encoded)) |applied| {
gpa.free(applied);
return false;
} else |_| {}
_ = log.putDelta(item.name, 1, encoded) catch return false;
self.verifier_fault_injected = true;
return true;
}
/// Replace every occurrence of one artifact name with another in a staging
/// text file. Equal lengths, so this is a straight in-place substitution.
fn substituteStagingName(
self: *CommandScript,
staging: std.Io.Dir,
file_name: []const u8,
old_name: []const u8,
new_name: []const u8,
) void {
const injected = self.verifier_fault_injected;
const body = self.readStagingText(staging, file_name) orelse return;
defer std.heap.page_allocator.free(body);
var cursor: usize = 0;
var replaced = false;
while (std.mem.indexOfPos(u8, body, cursor, old_name)) |at| {
@memcpy(body[at .. at + new_name.len], new_name);
cursor = at + new_name.len;
replaced = true;
}
if (!replaced) return;
self.writeStagingText(staging, file_name, body);
// `writeStagingText` reports its own success; this helper only carries
// a rename that a real injection already made, so it must not be what
// turns the flag on.
self.verifier_fault_injected = injected;
}
/// Reparse a check-in and re-emit it with `fault` applied. Returns null for
/// anything that is not a check-in, so the caller can keep walking.
///
/// The replacement is stored under its OWN correct name -- the point of
/// both faults is that the store stays internally honest and only its
/// meaning changed.
fn mutatedCheckin(self: *CommandScript, item: StagedArtifact, fault: VerifierFault) ?StagedArtifact {
_ = self;
const gpa = std.heap.page_allocator;
var manifest = artifact.parse(gpa, item.bytes) catch return null;
defer manifest.deinit();
if (manifest.kind != .manifest) return null;
switch (fault) {
.tree_change_preserving_artifacts => {
if (manifest.f.len < 2) return null;
// Every blob still present and still named; two of them now
// hang at each other's paths.
const cards = @constCast(manifest.f);
std.mem.swap(?[]const u8, &cards[0].hash, &cards[cards.len - 1].hash);
},
.parent_reorder => {
if (manifest.p.len == 0) return null;
const card = manifest.p[0];
if (card.count < 2) return null;
var parts = std.mem.splitScalar(u8, card.parents orelse return null, ' ');
const first = parts.next() orelse return null;
const second = parts.next() orelse return null;
const swapped = std.fmt.allocPrint(gpa, "{s} {s}", .{ second, first }) catch return null;
defer gpa.free(swapped);
const cards = @constCast(manifest.p);
cards[0].parents = swapped;
const bytes = artifact.create(&manifest, gpa) catch return null;
var name_buf: [64]u8 = undefined;
const name = artifact.artifactName(git_import.import_hash_algo, bytes, &name_buf);
var raw: [32]u8 = undefined;
const digest = rawHashName(name, &raw) orelse {
gpa.free(bytes);
return null;
};
return .{ .name = gpa.dupe(u8, digest) catch return null, .bytes = bytes };
},
else => return null,
}
const bytes = artifact.create(&manifest, gpa) catch return null;
var name_buf: [64]u8 = undefined;
const name = artifact.artifactName(git_import.import_hash_algo, bytes, &name_buf);
var raw: [32]u8 = undefined;
const digest = rawHashName(name, &raw) orelse {
gpa.free(bytes);
return null;
};
return .{ .name = gpa.dupe(u8, digest) catch return null, .bytes = bytes };
}
/// Whether the staging sibling holds `name` right now.
fn stagingHolds(self: *CommandScript, name: []const u8) bool {
const parent = self.destination_parent orelse return false;
const io = self.io orelse return false;
// Reopened for iteration: Linux enforces iterate rights at readdir
// time (a plain handle is O_PATH there); macOS does not.
var it_dir = parent.openDir(io, ".", .{ .iterate = true }) catch return false;
defer it_dir.close(io);
var entries = it_dir.iterate();
while (entries.next(io) catch return false) |entry| {
if (!std.mem.startsWith(u8, entry.name, ".zit-tmp-import-")) continue;
var staging = parent.openDir(io, entry.name, .{ .follow_symlinks = false }) catch return false;
defer staging.close(io);
return fileExists(io, staging, name);
}
return false;
}
/// Whether a staging sibling OTHER than a deliberately planted replacement
/// survived the run. Read after `run` returns, so a true answer means this
/// invocation leaked one.
///
/// Fails CLOSED: an unreadable parent reports a leak. This backs a
/// negative assertion -- `!temporary_sibling_left` -- so answering "no
/// leak" on a question that could not be asked is how a leak assertion
/// passes vacuously, which is the defect this field was added to fix.
fn stagingSiblingSurvives(self: *CommandScript) bool {
const parent = self.destination_parent orelse return true;
const io = self.io orelse return true;
const planted = self.stagingReplacementName();
var it_dir = parent.openDir(io, ".", .{ .iterate = true }) catch return true;
defer it_dir.close(io);
var entries = it_dir.iterate();
while (entries.next(io) catch return true) |entry| {
if (!std.mem.startsWith(u8, entry.name, ".zit-tmp-import-")) continue;
// A script that replaces the staging entry at the publication
// boundary leaves its plant behind BY DESIGN -- the transaction
// correctly refuses to delete an entry it no longer owns. Excluding
// that one name keeps the assertion live for every other sibling,
// rather than disabling it wholesale in exactly the shape where a
// leak is most plausible.
if (planted.len != 0 and std.mem.eql(u8, entry.name, planted)) continue;
return true;
}
return false;
}
fn replaceStaging(context: *anyopaque, parent: std.Io.Dir, name: []const u8) void {
const self: *CommandScript = @ptrCast(@alignCast(context));
const replacement = self.replace_staging_at_publish orelse return;
const io = self.io orelse return;
if (name.len > self.staging_replacement_name.len) return;
parent.deleteTree(io, name) catch return;
switch (replacement) {
.directory => {
parent.createDir(io, name, .default_dir) catch return;
var directory = parent.openDir(io, name, .{ .follow_symlinks = false }) catch return;
defer directory.close(io);
var marker = directory.createFile(io, "replacement-sentinel", .{ .read = true, .exclusive = true }) catch return;
defer marker.close(io);
marker.writeStreamingAll(io, "survives\n") catch return;
},
.symlink => parent.symLink(io, "outside", name, .{}) catch return,
}
@memcpy(self.staging_replacement_name[0..name.len], name);
self.staging_replacement_name_len = name.len;
self.staging_replacement_observed = true;
}
pub fn stagingReplacementName(self: *const CommandScript) []const u8 {
return self.staging_replacement_name[0..self.staging_replacement_name_len];
}
/// Derived from `exportArgv` so this fake cannot drift from the command
/// the importer actually runs. Shows the default signature policy; the
/// opt-out's effect is covered directly against `exportArgv`.
pub fn exportCommandLine(_: *const CommandScript) []const u8 {
return comptime line: {
const argv = exportArgv("fixture.git", false);
var joined: []const u8 = argv[0];
for (1..argv.len) |index| joined = joined ++ " " ++ argv[index];
break :line joined;
};
}
};
/// Caller-owned capture storage for `runScripted`'s stdout/stderr. The
/// returned `ScriptResult` slices point INTO this struct, so it must outlive
/// every read of the result — the storage used to live on `runScripted`'s own
/// frame, which made the returned slices dangle (visible as garbled stderr on
/// Linux, silent luck on macOS).
pub const ScriptCapture = struct {
out: [4096]u8 = undefined,
err: [4096]u8 = undefined,
};
pub const ScriptResult = struct {
code: cli.Exit,
out: []const u8,
err: []const u8,
lifecycle: ChildLifecycle,
retained_diagnostic: []const u8,
published_durability_uncertain: bool = false,
};
pub fn runScripted(
gpa: std.mem.Allocator,
io: std.Io,
request: Request,
script: *CommandScript,
hooks: PublicationHooks,
capture: *ScriptCapture,
) ScriptResult {
var out = std.Io.Writer.fixed(&capture.out);
var err = std.Io.Writer.fixed(&capture.err);
const streams: cli.Streams = .{ .out = &out, .err = &err };
script.preflight_starts += 1;
if (script.preflight == .git_missing or script.preflight == .not_repository or script.preflight == .missing_objects or script.preflight == .shallow or script.preflight == .partial_promisor or (script.preflight == .detached_head and script.default_branch == null)) {
script.lifecycle.reaped += 1;
return .{ .code = .failure, .out = out.buffered(), .err = err.buffered(), .lifecycle = script.lifecycle, .retained_diagnostic = "preflight rejected" };
}
if (script.preflight == .detached_head) script.detached_head_recorded = true;
script.destination_parent = request.destination_parent;
script.destination_name = request.destination_name;
script.io = io;
var effective_request = request;
effective_request.default_branch = script.default_branch;
const code = run(gpa, io, effective_request, streams, script.dependencies(hooks));
// Asked after the run, from outside. `stagingSiblingSurvives` excludes a
// deliberately planted replacement by name, so this stays a live leak
// assertion even on the staging-replacement scripts.
script.temporary_sibling_left = script.stagingSiblingSurvives();
if (code == .failure and hooks.fail_at == .sync_parent_after_publish) {
return .{ .code = code, .out = out.buffered(), .err = err.buffered(), .lifecycle = script.lifecycle, .retained_diagnostic = "durability uncertain", .published_durability_uncertain = true };
}
return .{ .code = code, .out = out.buffered(), .err = err.buffered(), .lifecycle = script.lifecycle, .retained_diagnostic = "scripted diagnostic" };
}
pub fn dispatchForTest(args: []const []const u8) ScriptResult {
if (args.len == 1 and std.mem.eql(u8, args[0], "--help")) {
// Rendered from `cli.import_git_usage`, not transcribed: the hand copy
// this replaced had silently fallen two options behind the real help.
// The `usage: ` prefix is `main.zig`'s, carried here so this stub's
// output is a true PREFIX of what the real help prints rather than
// merely sharing its option surface.
return .{ .code = .ok, .out = "usage: " ++ cli.import_git_usage ++ "\n", .err = "", .lifecycle = .{}, .retained_diagnostic = "" };
}
_ = cli.parseImportGitArgs(args) catch {
return .{ .code = .usage, .out = "", .err = "invalid import-git arguments\n", .lifecycle = .{}, .retained_diagnostic = "" };
};
return .{ .code = .failure, .out = "", .err = "test dispatch requires scripted dependencies\n", .lifecycle = .{}, .retained_diagnostic = "" };
}
test "import heap reuses a freed block for its class, passes large requests through, and returns every block at the end" {
var heap: ImportHeap = .{ .backing = std.testing.allocator };
defer heap.deinit();
const gpa = heap.caching();
// A freed block serves the next request of its class, whatever its size.
const first = try gpa.alloc(u8, 100);
const first_ptr = first.ptr;
gpa.free(first);
const again = try gpa.alloc(u8, 120);
try std.testing.expectEqual(first_ptr, again.ptr);
// A different class does not take it.
const other = try gpa.alloc(u8, 129);
try std.testing.expect(other.ptr != again.ptr);
gpa.free(other);
gpa.free(again);
// Growth within a class stays in place; across classes it moves.
var list: std.ArrayList(u64) = .empty;
for (0..10_000) |i| try list.append(gpa, i);
for (list.items, 0..) |value, i| try std.testing.expectEqual(@as(u64, i), value);
list.deinit(gpa);
// Past the largest class the backing allocator answers directly.
const large = try gpa.alloc(u8, (1 << ImportHeap.max_shift) + 1);
gpa.free(large);
try std.testing.expect(heap.free_lists[ImportHeap.class_count - 1] != null);
// An over-aligned request lands in a class at least as large as its alignment.
const aligned = try gpa.alignedAlloc(u8, .fromByteUnits(4096), 16);
try std.testing.expect(std.mem.isAligned(@intFromPtr(aligned.ptr), 4096));
gpa.free(aligned);
}
test "import heap is a pass-through over any allocator but the SMP one" {
var heap: ImportHeap = .{ .backing = std.testing.allocator };
defer heap.deinit();
try std.testing.expectEqual(std.testing.allocator.vtable, heap.allocator().vtable);
}