//! `import-git --with-submodules`: import each submodule beside the parent
//! first, so the parent's ledger can pin the check-ins they became.
//!
//! The submodules are found in the source's own history: every `.gitmodules`
//! any commit held, so a module dropped before the tip is still found. Each
//! distinct url is mirrored with Git and imported into a sibling of DEST,
//! `<name>.zit`, or added to with `--into` when that sibling is already an
//! import. The siblings' commit maps then say, for any Git commit a gitlink
//! pins, which zit check-in it is; the parent import writes those pins as
//! `zit` lines naming the sibling by `../<name>.zit`.
//!
//! A module that cannot be mirrored or imported is reported and left out; its
//! pins stay `git` lines. The parent import never refuses because of a
//! neighbour, and nothing here changes a check-in the parent already holds.
const std = @import("std");
const zit = @import("zit");
const cli = zit.cli;
const modules_ledger = zit.modules_ledger;
const git_import = zit.git_import;
const map_file = zit.git_import_map;
/// Submodule commits by Git OID, and the url each sibling is named by.
pub const Targets = struct {
by_oid: std.StringHashMapUnmanaged(git_import.ModuleTarget) = .empty,
arena: std.heap.ArenaAllocator,
pub fn deinit(self: *Targets) void {
self.arena.deinit();
self.* = undefined;
}
};
/// Output a Git query may produce: a history's `.gitmodules` texts and
/// commit lists, not file content.
const max_git_output: usize = 64 << 20;
/// Most distinct submodule urls one import follows.
const max_modules: usize = 1024;
/// Import every submodule `source` names into a sibling of DEST, whose
/// parent directory is `parent_abs`, and return the commits they hold.
pub fn importSiblings(
gpa: std.mem.Allocator,
io: std.Io,
environ: *const std.process.Environ.Map,
source: []const u8,
parent_abs: []const u8,
streams: cli.Streams,
quiet: bool,
) error{OutOfMemory}!Targets {
var targets: Targets = .{ .arena = std.heap.ArenaAllocator.init(gpa) };
errdefer targets.deinit();
const a = targets.arena.allocator();
const urls = try moduleUrls(gpa, io, environ, a, source);
if (urls.len == 0) {
if (!quiet) streams.err.writeAll("import-git: --with-submodules: the source names no submodules\n") catch {};
return targets;
}
const base = try remoteBase(gpa, io, environ, a, source);
var names: std.StringHashMapUnmanaged(void) = .empty;
for (urls) |raw| {
const url = resolveModuleUrl(a, raw, base) catch |err| switch (err) {
error.OutOfMemory => return error.OutOfMemory,
error.NoBase => {
streams.err.print("import-git: submodule url '{s}' is relative and the source has no origin to resolve it against; its pins stay git lines\n", .{raw}) catch {};
continue;
},
};
const name = try uniqueName(a, &names, url);
const sibling = try std.fs.path.join(a, &.{ parent_abs, try std.mem.concat(a, u8, &.{ name, ".zit" }) });
if (!try importOne(gpa, io, environ, a, url, name, sibling, parent_abs, streams, quiet)) continue;
var dir = std.Io.Dir.cwd().openDir(io, sibling, .{}) catch continue;
defer dir.close(io);
var map = map_file.load(gpa, io, dir) catch {
streams.err.print("import-git: submodule {s}: the import map in '{s}' cannot be read; its pins stay git lines\n", .{ name, sibling }) catch {};
continue;
};
defer map.deinit();
const target_url = try std.mem.concat(a, u8, &.{ "../", name, ".zit" });
for (map.contents.commits) |commit| {
const oid = try a.dupe(u8, commit.oid);
const gop = try targets.by_oid.getOrPut(a, oid);
if (gop.found_existing) continue; // the same commit in two modules: the first url names it
gop.value_ptr.* = .{ .checkin = try a.dupe(u8, commit.checkin), .url = target_url };
}
if (!quiet) streams.err.print("import-git: submodule {s} from {s} is in {s}\n", .{ name, url, sibling }) catch {};
}
return targets;
}
/// Every distinct url any `.gitmodules` in `source`'s history names, in the
/// order first seen.
fn moduleUrls(gpa: std.mem.Allocator, io: std.Io, environ: *const std.process.Environ.Map, a: std.mem.Allocator, source: []const u8) error{OutOfMemory}![]const []const u8 {
const commits = (try runGit(gpa, io, environ, &.{ "git", "-C", source, "--no-lazy-fetch", "log", "--all", "--format=%H", "--", ".gitmodules" })) orelse return &.{};
defer gpa.free(commits);
var seen: std.StringHashMapUnmanaged(void) = .empty;
var urls: std.ArrayList([]const u8) = .empty;
var lines = std.mem.tokenizeScalar(u8, commits, '\n');
while (lines.next()) |commit| {
const spec = try std.mem.concat(gpa, u8, &.{ commit, ":.gitmodules" });
defer gpa.free(spec);
// A commit that deleted the file names it too; it has nothing to show.
const text = (try runGit(gpa, io, environ, &.{ "git", "-C", source, "--no-lazy-fetch", "show", spec })) orelse continue;
defer gpa.free(text);
var map = try modules_ledger.parseGitmodules(gpa, text);
defer map.deinit();
var it = map.paths.valueIterator();
while (it.next()) |url| {
if (seen.contains(url.*)) continue;
if (urls.items.len == max_modules) return urls.items;
const owned = try a.dupe(u8, url.*);
try seen.put(a, owned, {});
try urls.append(a, owned);
}
}
return urls.items;
}
/// What a relative submodule url resolves against: the source's `origin`
/// remote, else the source's own absolute path, as Git uses for a
/// repository with no remote.
fn remoteBase(gpa: std.mem.Allocator, io: std.Io, environ: *const std.process.Environ.Map, a: std.mem.Allocator, source: []const u8) error{OutOfMemory}!?[]const u8 {
if (try runGit(gpa, io, environ, &.{ "git", "-C", source, "config", "--get", "remote.origin.url" })) |out| {
defer gpa.free(out);
const url = std.mem.trim(u8, out, " \t\r\n");
if (url.len != 0) return try a.dupe(u8, url);
}
const cwd = (cli.currentPathAlloc(gpa, io) catch return null) orelse return null;
defer gpa.free(cwd);
return try std.fs.path.resolve(a, &.{ cwd, source });
}
/// A `./` or `../` url against `base`, Git's way: each `../` drops one
/// segment. A url with a scheme or an absolute path is used as it is.
pub fn resolveModuleUrl(a: std.mem.Allocator, url: []const u8, base: ?[]const u8) error{ OutOfMemory, NoBase }![]const u8 {
if (!std.mem.startsWith(u8, url, "./") and !std.mem.startsWith(u8, url, "../")) return url;
var resolved = std.mem.trimEnd(u8, base orelse return error.NoBase, "/");
var rel = url;
// Bounded by the url's length: each pass consumes at least two bytes.
while (true) {
if (std.mem.startsWith(u8, rel, "./")) {
rel = rel[2..];
} else if (std.mem.startsWith(u8, rel, "../")) {
rel = rel[3..];
const slash = std.mem.lastIndexOfScalar(u8, resolved, '/') orelse return error.NoBase;
resolved = resolved[0..slash];
if (resolved.len == 0 or std.mem.endsWith(u8, resolved, "/")) return error.NoBase;
} else break;
}
return std.mem.concat(a, u8, &.{ resolved, "/", rel });
}
/// The sibling's name: the url's last segment without `.git`, made unique
/// in this import, and reduced to characters a directory name can hold.
fn uniqueName(a: std.mem.Allocator, names: *std.StringHashMapUnmanaged(void), url: []const u8) error{OutOfMemory}![]const u8 {
var last = std.mem.trimEnd(u8, url, "/");
if (std.mem.lastIndexOfAny(u8, last, "/:")) |i| last = last[i + 1 ..];
if (std.mem.endsWith(u8, last, ".git")) last = last[0 .. last.len - 4];
const clean = try a.alloc(u8, @max(last.len, 1));
if (last.len == 0) clean[0] = 'm';
for (last, 0..) |c, i| clean[i] = if (std.ascii.isAlphanumeric(c) or c == '-' or c == '_' or c == '.') c else '-';
if (clean[0] == '.') clean[0] = '-';
var name: []const u8 = clean;
var n: usize = 2;
while (names.contains(name)) : (n += 1) name = try std.fmt.allocPrint(a, "{s}-{d}", .{ clean, n });
try names.put(a, name, {});
return name;
}
/// Mirror `url` and import it into `sibling`. True when the sibling now
/// holds the module's history.
fn importOne(
gpa: std.mem.Allocator,
io: std.Io,
environ: *const std.process.Environ.Map,
a: std.mem.Allocator,
url: []const u8,
name: []const u8,
sibling: []const u8,
parent_abs: []const u8,
streams: cli.Streams,
quiet: bool,
) error{OutOfMemory}!bool {
const mirror = try std.fs.path.join(a, &.{ parent_abs, try std.mem.concat(a, u8, &.{ ".zit-module-", name, ".git" }) });
std.Io.Dir.cwd().deleteTree(io, mirror) catch {};
defer std.Io.Dir.cwd().deleteTree(io, mirror) catch {};
if ((try runGit(gpa, io, environ, &.{ "git", "clone", "--mirror", "--quiet", url, mirror })) == null) {
streams.err.print("import-git: submodule {s}: cannot fetch {s}; its pins stay git lines\n", .{ name, url }) catch {};
return false;
}
const exists = blk: {
var dir = std.Io.Dir.cwd().openDir(io, sibling, .{}) catch break :blk false;
defer dir.close(io);
_ = dir.statFile(io, map_file.map_filename, .{}) catch break :blk false;
break :blk true;
};
const exe = std.process.executablePathAlloc(io, gpa) catch return false;
defer gpa.free(exe);
const argv: []const []const u8 = if (exists)
&.{ exe, "import-git", mirror, sibling, "--into", "--quiet" }
else
&.{ exe, "import-git", mirror, sibling, "--quiet" };
var child = std.process.spawn(io, .{ .argv = argv, .environ_map = environ, .stdout = .ignore }) catch return false;
const term = child.wait(io) catch return false;
const ok = switch (term) {
.exited => |code| code == 0,
else => false,
};
if (!ok) streams.err.print("import-git: submodule {s}: importing {s} into {s} failed; its pins stay git lines\n", .{ name, url, sibling }) catch {};
if (!quiet and ok) streams.err.print("import-git: submodule {s}: {s} {s}\n", .{ name, if (exists) "added to" else "imported into", sibling }) catch {};
return ok;
}
/// Git's stdout, or null when Git exits nonzero (a commit without the file,
/// a url that cannot be fetched).
fn runGit(gpa: std.mem.Allocator, io: std.Io, environ: *const std.process.Environ.Map, argv: []const []const u8) error{OutOfMemory}!?[]u8 {
const result = std.process.run(gpa, io, .{
.argv = argv,
.environ_map = environ,
.stdout_limit = .limited(max_git_output),
.stderr_limit = .limited(1 << 16),
}) catch |err| switch (err) {
error.OutOfMemory => return error.OutOfMemory,
else => return null,
};
gpa.free(result.stderr);
switch (result.term) {
.exited => |code| if (code == 0) return result.stdout,
else => {},
}
gpa.free(result.stdout);
return null;
}
test "submodule urls resolve against the superproject's remote, and names stay unique" {
var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator);
defer arena_state.deinit();
const a = arena_state.allocator();
try std.testing.expectEqualStrings("https://h/o/lib.git", try resolveModuleUrl(a, "../lib.git", "https://h/o/app.git"));
try std.testing.expectEqualStrings("/srv/git/lib", try resolveModuleUrl(a, "../lib", "/srv/git/app"));
try std.testing.expectEqualStrings("git@h:o/lib.git", try resolveModuleUrl(a, "git@h:o/lib.git", null));
try std.testing.expectError(error.NoBase, resolveModuleUrl(a, "../lib", null));
var names: std.StringHashMapUnmanaged(void) = .empty;
try std.testing.expectEqualStrings("lib", try uniqueName(a, &names, "https://h/o/lib.git"));
try std.testing.expectEqualStrings("lib-2", try uniqueName(a, &names, "https://other/lib.git/"));
try std.testing.expectEqualStrings("my-lib", try uniqueName(a, &names, "git@h:o/my lib.git"));
}