//! `zit import-git SOURCE DEST --into`: add the upstream commits a destination
//! does not yet hold to the ones it has.
//!
//! The fresh import in `git_import_cli.zig` builds a whole store in a staging
//! directory and publishes it with one rename. That cannot apply here without
//! copying the destination, which for the repositories this exists for is the
//! cost being avoided. Instead the destination's own log is opened for append,
//! the way a pull appends to it, and the commit point is the log's own: frames
//! are written, then synced, then the map that names them is written. Any
//! failure before the map is written truncates the log back to the length it
//! had when the import began, so a failed run leaves the destination as it
//! found it, map included.
//!
//! What the destination already holds comes from the map (`git_import_map`):
//! every imported commit's OID, check-in name, branch and first-parent depth,
//! and every ref's tip. The export excludes those tips, names an excluded
//! parent by OID, and the importer resolves the name through the seeded map
//! rather than the stream.
const std = @import("std");
const zit = @import("zit");
const cli = zit.cli;
const logstore = zit.logstore;
const storage = zit.storage;
const fast_export = zit.git_fast_export;
const git_import = zit.git_import;
const import_cli = @import("git_import_cli.zig");
const map_file = zit.git_import_map;
const LogTreeStore = zit.log_tree_store.LogTreeStore;
const PublicationHooks = zit.publication.PublicationHooks;
pub fn run(
gpa: std.mem.Allocator,
io: std.Io,
request: import_cli.Request,
streams: cli.Streams,
dependencies: import_cli.Dependencies,
) cli.Exit {
if (request.provenance) {
streams.err.writeAll("import-git: --provenance applies to a fresh import; an import that adds to a destination keeps no per-object record\n") catch {};
return .failure;
}
var destination = request.destination_parent.openDir(io, request.destination_name, .{}) catch {
streams.err.print("import-git: destination '{s}' does not exist; omit --into to create it\n", .{request.destination_name}) catch {};
return .failure;
};
defer destination.close(io);
var map = map_file.load(gpa, io, destination) catch |err| {
switch (err) {
error.Absent => streams.err.print(
"import-git: '{s}' has no commit map, so nothing says which upstream commits it holds; it was not written by this verb, or by a zit older than the map. Import into a fresh destination instead\n",
.{request.destination_name},
) catch {},
else => streams.err.print("import-git: the commit map of '{s}' cannot be read ({s})\n", .{ request.destination_name, @errorName(err) }) catch {},
}
return .failure;
};
defer map.deinit();
if (map.contents.mirror_classifier_fired) {
streams.err.writeAll("import-git: the first import of this destination met a Fossil mirror's own metadata; adding to such a destination is not supported, re-import it instead\n") catch {};
return .failure;
}
if (settingsConflict(request, map.contents.settings)) |message| {
streams.err.print("import-git: {s}\n", .{message}) catch {};
return .failure;
}
const describe = dependencies.git.vtable.describe_source orelse {
streams.err.writeAll("import-git: cannot identify the source, so cannot tell whether it is the one this destination was imported from\n") catch {};
return .failure;
};
var identity = describe(dependencies.git.context, gpa, request.source) catch |err| {
if (dependencies.git.vtable.startup_diagnostics) |write_diagnostics| write_diagnostics(dependencies.git.context, streams.err);
streams.err.print("import-git: cannot identify the source: {s}\n", .{@errorName(err)}) catch {};
return .failure;
};
defer identity.deinit(gpa);
if (!sameSource(identity, map.contents)) {
streams.err.print("import-git: '{s}' is not the repository '{s}' was imported from: a root commit the destination holds is not in its history\n", .{ request.source, request.destination_name }) catch {};
return .failure;
}
var store_append = cli.openStoreAppend(io, destination) catch |err| {
if (err != error.StoreBusy) streams.err.print("import-git: cannot open the destination's store for writing: {s}\n", .{@errorName(err)}) catch {};
return .failure;
};
var store_live = true;
defer if (store_live) store_append.close(io);
const log_stat = store_append.file.stat(io) catch |err| {
streams.err.print("import-git: cannot read the destination's store: {s}\n", .{@errorName(err)}) catch {};
return .failure;
};
var file_storage = storage.FileStorage.init(io, store_append.file, log_stat.size);
var log = logstore.LogStore.open(gpa, file_storage.storageFile(), 0) catch |err| {
streams.err.print("import-git: cannot open the destination's store: {s}\n", .{@errorName(err)}) catch {};
return .failure;
};
var log_live = true;
defer if (log_live) log.deinit(gpa);
if (log.artifactCount() != map.contents.artifact_count) {
streams.err.print("import-git: '{s}' has been written to since its last import (a commit, a sync, or an import that was interrupted after writing), so its commit map no longer describes it; it cannot be added to\n", .{request.destination_name}) catch {};
return .failure;
}
const start_len = log.size();
var outcome = importMore(gpa, io, request, &map, &identity, &log, destination, streams, dependencies) catch |err| {
if (err == error.DurabilityUncertain) {
// The new map is already in place over the committed log, so
// nothing may be undone; only the sync of the rename is in doubt.
streams.err.print("import-git: '{s}' was updated but its directory could not be synced, so the update may not survive a power loss until the next sync\n", .{request.destination_name}) catch {};
return .failure;
}
// Unwind: the frames appended since `start_len` are cut off, so the
// store is byte-for-byte what it was, and the old map still names it.
log.deinit(gpa);
log_live = false;
file_storage.storageFile().truncate(start_len) catch {
streams.err.print("import-git: import failed ({s}) and the store could not be restored to its previous length; run 'zit verify'\n", .{@errorName(err)}) catch {};
return .failure;
};
store_append.file.sync(io) catch {};
if (err == error.InjectedPublicationFailure) {
streams.err.writeAll("import-git: import failed: InjectedPublicationFailure; the destination is unchanged\n") catch {};
} else if (err == error.MirrorMetadataInNewCommits) {
streams.err.writeAll("import-git: the new commits carry a Fossil mirror's own metadata, which an import that adds to a destination cannot classify; the destination is unchanged. Re-import it from scratch instead\n") catch {};
} else {
streams.err.print("import-git: import failed: {s}; the destination is unchanged\n", .{@errorName(err)}) catch {};
}
return .failure;
};
defer outcome.deinit(gpa);
log.deinit(gpa);
log_live = false;
store_append.close(io);
store_live = false;
// Pre-pay the derived-state cache against the published store, as the
// fresh import does; best-effort like every cache write.
zit.Repository.warmCache(gpa, io, destination);
if (!request.quiet) {
streams.err.print("import-git: added {d} check-in(s) to '{s}'\n", .{ outcome.added_commits, request.destination_name }) catch {};
for (outcome.notes.items) |note| streams.err.print("import-git: {s}\n", .{note}) catch {};
}
return .ok;
}
/// A flag that contradicts what the first import recorded, as a message, or
/// null when the request is compatible. An absent flag means "reuse".
fn settingsConflict(request: import_cli.Request, recorded: map_file.Settings) ?[]const u8 {
switch (request.submanifest) {
.flat => if (recorded.submanifest != .flat) return "--no-submanifests contradicts the destination, which was imported with sub-manifests; omit it",
.tree => |pin| if (pin) |k| switch (recorded.submanifest) {
.flat => return "--submanifest-k contradicts the destination, which was imported flat; omit it",
.tree => |recorded_k| if (recorded_k != k) return "--submanifest-k contradicts the k the destination was imported with; omit it",
},
}
const requested_attribution: map_file.Attribution = switch (request.attribution) {
.committer => .committer,
.author => .author,
};
if (requested_attribution != recorded.attribution) return "--attribution contradicts the identities the destination was imported with; omit it";
if (request.refuse_signed != recorded.refuse_signed) {
return if (request.refuse_signed)
"--refuse-signed contradicts the destination, which was imported with signatures stripped; omit it"
else
"the destination was imported with --refuse-signed; pass it again";
}
if (request.default_branch) |requested| {
if (!std.mem.eql(u8, requested, recorded.default_branch)) return "--default-branch contradicts the main branch the destination was imported with; omit it";
}
return null;
}
/// The source is the one the destination came from when every root the
/// destination recorded is still a root of the source. The source may have
/// gained roots since (an orphan branch, a first notes commit); it cannot
/// have lost one and still be the history the destination holds.
fn sameSource(identity: import_cli.SourceIdentity, recorded: map_file.Contents) bool {
const format: map_file.ObjectFormat = if (std.mem.eql(u8, identity.object_format, "sha256")) .sha256 else .sha1;
if (format != recorded.object_format) return false;
for (recorded.roots) |theirs| {
const present = for (identity.roots) |mine| {
if (std.mem.eql(u8, mine, theirs)) break true;
} else false;
if (!present) return false;
}
return true;
}
const Outcome = struct {
added_commits: usize,
notes: std.ArrayList([]u8),
fn deinit(self: *Outcome, gpa: std.mem.Allocator) void {
for (self.notes.items) |note| gpa.free(note);
self.notes.deinit(gpa);
self.* = undefined;
}
};
/// Everything between the store being open for append and the new map being
/// in place. Any error unwinds in `run`.
fn importMore(
gpa: std.mem.Allocator,
io: std.Io,
request: import_cli.Request,
map: *const map_file.Map,
identity: *const import_cli.SourceIdentity,
log: *logstore.LogStore,
destination: std.Io.Dir,
streams: cli.Streams,
dependencies: import_cli.Dependencies,
) !Outcome {
const recorded = map.contents;
const hooks = dependencies.publication;
// The tips to exclude: every ref's peeled commit, each once.
var tips: std.ArrayList([]const u8) = .empty;
defer tips.deinit(gpa);
var tip_set: std.StringHashMapUnmanaged(void) = .empty;
defer tip_set.deinit(gpa);
for (recorded.refs) |ref| {
const seen = try tip_set.getOrPut(gpa, ref.oid);
if (seen.found_existing) continue;
try tips.append(gpa, ref.oid);
}
var pipes = try dependencies.git.vtable.start_export(dependencies.git.context, request.source, tips.items);
var session_finished = false;
defer if (!session_finished) {
_ = import_cli.finishSession(&pipes, streams, dependencies.git, request.quiet, true) catch false;
};
var tree_store = LogTreeStore{ .gpa = gpa, .log = log };
defer tree_store.deinitCache();
var importer = try git_import.Importer.init(gpa, log, tree_store.treeStore(), .{
.submanifest = switch (recorded.settings.submanifest) {
.flat => .flat,
.tree => |k| .{ .tree = k },
},
.attribution = switch (recorded.settings.attribution) {
.committer => .committer,
.author => .author,
},
});
defer importer.deinit();
for (recorded.commits) |commit| {
try importer.seedKnownCommit(commit.oid, commit.checkin, commit.branch, commit.depth, commit.height);
}
var reader = fast_export.Reader.init(gpa, pipes.stdout, .{
.max_line = 1024 * 1024,
.max_data = 1024 * 1024 * 1024,
.max_path = 4096,
.max_events = std.math.maxInt(u64),
.max_total_data = dependencies.limits.max_export_payload_bytes,
.max_command_arena_bytes = dependencies.limits.max_command_arena_bytes,
});
defer reader.deinit();
var diagnostic: import_cli.LimitDiagnostic = .{};
import_cli.consumeExport(&reader, &importer, dependencies.limits, &diagnostic, .{
.streams = streams,
.io = io,
.quiet = request.quiet,
.commit_total = pipes.commit_total,
}) catch |err| {
const scripted_child_failed = pipes.exit_code_ref == null and import_cli.terminalExitCode(&pipes) != 0;
const child_finished = import_cli.finishSession(&pipes, streams, dependencies.git, request.quiet, true) catch |lifecycle_err| {
session_finished = true;
return lifecycle_err;
};
session_finished = true;
diagnostic.write(streams.err);
if (scripted_child_failed or (child_finished and import_cli.terminalExitCode(&pipes) != 0)) return error.GitExportFailed;
return err;
};
_ = import_cli.finishSession(&pipes, streams, dependencies.git, request.quiet, false) catch |err| {
session_finished = true;
return err;
};
session_finished = true;
if (import_cli.terminalExitCode(&pipes) != 0) return error.GitExportFailed;
// The classifier's per-commit state is inherited along first parents,
// which the stream no longer carries, so its verdicts cannot be trusted
// here; a fresh import of the whole history is the only honest answer.
if (importer.mirrorCensus().commits_with_candidates != 0) return error.MirrorMetadataInNewCommits;
// The primary label is the one the destination already carries, read
// off the recorded main tip rather than recomputed from the source's
// branch names.
const primary_ref = try std.fmt.allocPrint(gpa, "refs/heads/{s}", .{recorded.settings.default_branch});
defer gpa.free(primary_ref);
const primary_label: []const u8 = label: {
const old_main = map.lookupRef(primary_ref) orelse break :label git_import.fossil_main_branch;
const old_tip = map.lookupOid(old_main.oid) orelse break :label git_import.fossil_main_branch;
break :label old_tip.branch;
};
var finalized = try importer.finalizeRefs(.{
.default_branch = recorded.settings.default_branch,
.primary_label = primary_label,
.symbolic_controls = false,
});
defer finalized.deinit(gpa);
if (recorded.settings.submanifest == .flat and finalized.stored_submanifest) return error.SubmanifestEmittedUnderFlatImport;
const added = importer.commitRecordCount();
var outcome: Outcome = .{ .added_commits = added, .notes = .empty };
errdefer outcome.deinit(gpa);
for (finalized.controls) |control| try import_cli.putSha1Artifact(log, control);
// Names are reconciled against what the source publishes now, not the
// stream: a ref on a commit the destination already holds never appears
// in the stream, an annotated tag on one is dropped from it outright.
// Every control written here is dated now, to the millisecond, so a name
// that moves, or returns to a check-in it once left, outranks the control
// before it even within one second; a name that did not change writes
// nothing. The main branch's own `sym-` name is never touched: it
// propagates from the branch control, and cancelling it on a tip the
// head left would take the branch itself off that check-in.
var now_buf: [23]u8 = undefined;
const now_ns = std.Io.Timestamp.now(io, .real).nanoseconds;
const now = cli.formatWriterTimestampMs(@intCast(@divFloor(now_ns, std.time.ns_per_s)), @intCast(@divFloor(@mod(now_ns, std.time.ns_per_s), std.time.ns_per_ms)), &now_buf);
var refs: std.ArrayList(map_file.RefRecord) = .empty;
defer refs.deinit(gpa);
var upstream_names: std.StringHashMapUnmanaged(void) = .empty;
defer upstream_names.deinit(gpa);
for (identity.refs) |ref| {
try upstream_names.put(gpa, ref.name, {});
const target = checkinOf(map, &importer, ref.oid) orelse {
const note = try std.fmt.allocPrint(gpa, "ref '{s}' points at a commit this import does not hold and was left out", .{ref.name});
errdefer gpa.free(note);
try outcome.notes.append(gpa, note);
continue;
};
try refs.append(gpa, .{ .name = ref.name, .oid = ref.oid });
const old = map.lookupRef(ref.name);
if (old) |before| if (std.mem.eql(u8, before.oid, ref.oid)) continue;
const canonical = try git_import.canonicalRefName(gpa, ref.name);
defer gpa.free(canonical);
if (old) |before| if (map.lookupOid(before.oid)) |left| {
try putSymbolic(log, &importer, left.checkin, canonical, .cancel, now);
if (std.mem.startsWith(u8, ref.name, "refs/heads/") and !firstParentLineReaches(&importer, target, left.checkin)) {
const note = try std.fmt.allocPrint(gpa, "head '{s}' moved to {s}; the check-ins it left behind stay", .{ ref.name["refs/heads/".len..], target[0..@min(target.len, 12)] });
errdefer gpa.free(note);
try outcome.notes.append(gpa, note);
}
};
try putSymbolic(log, &importer, target, canonical, .add, now);
// The main branch's own name propagates from its branch control, so
// it needs no help unless the head landed on a check-in another
// branch already labelled; then only a direct tag names it there.
if (std.mem.eql(u8, ref.name, primary_ref)) if (map.lookupOid(ref.oid)) |held| if (!std.mem.eql(u8, held.branch, primary_label)) {
const alias = try std.fmt.allocPrint(gpa, "sym-{s}", .{primary_label});
defer gpa.free(alias);
try putSymbolic(log, &importer, target, alias, .add, now);
};
}
for (recorded.refs) |old| {
if (upstream_names.contains(old.name)) continue;
const left = map.lookupOid(old.oid) orelse continue;
const canonical = try git_import.canonicalRefName(gpa, old.name);
defer gpa.free(canonical);
try putSymbolic(log, &importer, left.checkin, canonical, .cancel, now);
if (std.mem.startsWith(u8, old.name, "refs/heads/")) {
const note = try std.fmt.allocPrint(gpa, "head '{s}' is gone upstream; its check-ins stay", .{old.name["refs/heads/".len..]});
errdefer gpa.free(note);
try outcome.notes.append(gpa, note);
}
}
// Trees were stored for the first time: the latch the fresh import sets
// from its own emission has to follow, or the store lies about itself.
if (finalized.stored_submanifest) {
var current = (try log.readConfig(gpa)) orelse return error.ConfigMissing;
defer current.deinit(gpa);
if (!current.config.seen_submanifest) {
var updated = current.config;
updated.seen_submanifest = true;
try log.putConfig(updated);
}
}
try hooks.check(.before_into_commit);
try log.commit();
try hooks.check(.before_into_map);
// The new map: every commit recorded before plus the ones added, every
// ref the source publishes now.
var commits = try gpa.alloc(map_file.CommitRecord, recorded.commits.len + added);
defer gpa.free(commits);
@memcpy(commits[0..recorded.commits.len], recorded.commits);
for (0..added) |index| {
const record = try importer.commitRecord(index);
commits[recorded.commits.len + index] = .{
.oid = record.oid,
.checkin = record.hash,
.branch = record.branch,
.depth = record.depth,
.height = record.tree_height,
};
}
try map_file.write(gpa, io, destination, .{
.object_format = recorded.object_format,
.roots = recorded.roots,
.settings = recorded.settings,
.mirror_classifier_fired = false,
.artifact_count = log.artifactCount(),
.commits = commits,
.refs = refs.items,
});
return outcome;
}
/// The check-in a Git commit became: one this import added, or one the
/// destination recorded.
fn checkinOf(map: *const map_file.Map, importer: *const git_import.Importer, oid: []const u8) ?[]const u8 {
if (map.lookupOid(oid)) |record| return record.checkin;
for (0..importer.commitRecordCount()) |index| {
const record = importer.commitRecord(index) catch continue;
if (std.mem.eql(u8, record.oid, oid)) return record.hash;
}
return null;
}
fn putSymbolic(log: *logstore.LogStore, importer: *git_import.Importer, target: []const u8, canonical: []const u8, sign: enum { add, cancel }, now: []const u8) !void {
const bytes = switch (sign) {
.add => try importer.makeSymbolicAdd(target, canonical, now),
.cancel => try importer.makeSymbolicCancel(target, canonical, now),
};
defer importer.gpa.free(bytes);
try import_cli.putSha1Artifact(log, bytes);
}
/// Whether walking first parents from the new object `tip` reaches the
/// check-in `target` before leaving the objects this import added. A head
/// that advanced along its own line passes; one that was reset or rewritten
/// does not. A merge of the old tip into a new line reads as a move, which
/// is a safe over-report.
fn firstParentLineReaches(importer: *const git_import.Importer, tip: []const u8, target: []const u8) bool {
var cursor = importer.objectIndexOfHash(tip) orelse return false;
while (true) {
const object = importer.objectAt(cursor);
if (object.first_parent_index) |parent| {
cursor = parent;
continue;
}
const first = object.first_parent orelse return false;
return std.mem.eql(u8, first, target);
}
}