//! `zit modules` — list, sync and pin the submodules `.zit-modules` names.
//!
//! The ledger (`modules_ledger.zig`) is a tracked file, so pins travel with
//! every check-in and every peer. These verbs act on it from a checkout: a
//! `zit` module is a nested zit checkout at its path, cloned from its url and
//! updated to its pin. A `git` module pins a Git commit, which zit cannot
//! check out; `import-git --with-submodules` turns such lines into `zit`
//! lines by importing the module beside the parent.
//!
//! Nothing recurses unless asked, and nothing commits or pushes a module: a
//! module is a repository of its own, changed and published on its own.
const std = @import("std");
const zit = @import("zit");
const cli = zit.cli;
const checkout = zit.checkout;
const logstore = zit.logstore;
const storage = zit.storage;
const modules_ledger = zit.modules_ledger;
const repocache = zit.repocache;
pub const help =
\\usage: zit modules [list]
\\ zit modules sync [--recursive [--depth=N]] [PATH...]
\\ zit modules pin PATH...
\\
\\The submodules this checkout's .zit-modules names. Each line pins one
\\module: its path, its kind (zit or git), where it lives, and the check-in
\\or commit it is pinned to.
\\
\\list (the default) prints each module's path, kind, pin and state: at
\\pin, off pin (and where the checkout is instead), not synced, or not a
\\zit checkout.
\\
\\sync clones each zit module that is not synced into its path and
\\updates every one to its pin, pulling first when the pin is not yet
\\held. A relative url resolves against this repository's saved remote, as
\\Git resolves it. In a horizon checkout the clone is a structural horizon
\\clone at the pin. A git module pins a Git commit zit cannot check out;
\\it is named and left alone, and re-importing the parent with
\\'import-git --with-submodules' makes it a zit module. PATH limits sync to
\\those modules. --recursive syncs the modules' own modules too, to N
\\levels (default 8), once per url.
\\
\\pin records each named module's current check-in as its pin in
\\.zit-modules, for the next commit. A module pinned for the first time
\\takes its checkout's saved remote as its url.
\\
\\Nothing here commits or pushes a module.
\\
\\options:
\\ --recursive sync the modules' own modules too
\\ --depth=N how many levels --recursive goes (default 8, at most 64)
\\ -h, --help show this help and exit
\\
;
const default_depth: u32 = 8;
const max_depth: u32 = 64;
pub fn runModules(
gpa: std.mem.Allocator,
io: std.Io,
arena: std.mem.Allocator,
verb_args: []const []const u8,
streams: cli.Streams,
amb: cli.Ambient,
) cli.Exit {
_ = amb;
const Sub = enum { list, sync, pin };
var sub: Sub = .list;
var rest = verb_args;
if (rest.len != 0) {
if (std.mem.eql(u8, rest[0], "-h") or std.mem.eql(u8, rest[0], "--help")) {
streams.out.writeAll(help) catch {};
return .ok;
}
if (std.meta.stringToEnum(Sub, rest[0])) |named| {
sub = named;
rest = rest[1..];
} else if (rest[0].len == 0 or rest[0][0] != '-') {
streams.err.print("zit modules: unknown subcommand '{s}'\n", .{rest[0]}) catch {};
streams.err.writeAll("try 'zit modules --help'\n") catch {};
return .usage;
}
}
var recursive = false;
var depth: u32 = default_depth;
var paths: std.ArrayList([]const u8) = .empty;
for (rest) |tok| {
if (std.mem.eql(u8, tok, "-h") or std.mem.eql(u8, tok, "--help")) {
streams.out.writeAll(help) catch {};
return .ok;
} else if (sub == .sync and std.mem.eql(u8, tok, "--recursive")) {
recursive = true;
} else if (sub == .sync and std.mem.startsWith(u8, tok, "--depth=")) {
depth = std.fmt.parseInt(u32, tok["--depth=".len..], 10) catch return cli.usageError(streams, "modules", .unknown_option, tok);
if (depth == 0 or depth > max_depth) return cli.usageError(streams, "modules", .unknown_option, tok);
} else if (tok.len != 0 and tok[0] == '-') {
return cli.usageError(streams, "modules", .unknown_option, tok);
} else if (sub == .list) {
return cli.usageError(streams, "modules", .extra_argument, tok);
} else {
paths.append(arena, std.mem.trimEnd(u8, tok, "/")) catch return oom(streams);
}
}
if (sub == .pin and paths.items.len == 0) return cli.usageError(streams, "modules", .no_paths, "");
const root = (cli.discoverRoot(arena, io) catch return oom(streams)) orelse {
streams.err.writeAll("zit modules: not in a zit checkout\n") catch {};
return .failure;
};
var root_dir = root.dir;
defer root_dir.close(io);
return switch (sub) {
.list => list(arena, io, root_dir, streams),
.sync => {
var seen: std.StringHashMapUnmanaged(void) = .empty;
var failures: usize = 0;
syncTree(gpa, io, arena, root_dir, root.abs, paths.items, if (recursive) depth else 1, &seen, streams, &failures) catch |err| switch (err) {
error.OutOfMemory => return oom(streams),
error.InvalidLedger => return .failure,
};
return if (failures == 0) .ok else .failure;
},
.pin => pin(gpa, io, arena, root_dir, paths.items, streams),
};
}
fn oom(streams: cli.Streams) cli.Exit {
streams.err.writeAll("zit modules: out of memory\n") catch {};
return .failure;
}
/// The ledger in `dir`'s working tree, or an empty one when there is none.
/// A ledger that does not parse is named with its line and refused.
fn readLedger(arena: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, streams: cli.Streams, where: []const u8) error{ OutOfMemory, InvalidLedger }![]const modules_ledger.Entry {
const bytes = dir.readFileAlloc(io, modules_ledger.name, arena, .limited(modules_ledger.max_entries * 4 * modules_ledger.max_field_bytes)) catch |err| switch (err) {
error.FileNotFound => return &.{},
error.OutOfMemory => return error.OutOfMemory,
else => {
streams.err.print("zit modules: cannot read {s}{s}: {s}\n", .{ where, modules_ledger.name, @errorName(err) }) catch {};
return error.InvalidLedger;
},
};
var diagnostic: modules_ledger.Diagnostic = .{};
const ledger = modules_ledger.parse(arena, bytes, &diagnostic) catch |err| switch (err) {
error.OutOfMemory => return error.OutOfMemory,
error.InvalidLedger => {
streams.err.print("zit modules: {s}{s} line {d}: {s}\n", .{ where, modules_ledger.name, diagnostic.line, diagnostic.reason }) catch {};
return error.InvalidLedger;
},
};
return ledger.entries;
}
pub const State = union(enum) {
at_pin,
/// The checkout's current check-in.
off_pin: []const u8,
not_synced,
not_a_checkout,
};
/// Where the module at `path` stands against `pin`. `version_buf` holds
/// the off-pin hash.
pub fn stateOf(io: std.Io, root_dir: std.Io.Dir, path: []const u8, pin_hash: []const u8, version_buf: []u8) State {
var dir = root_dir.openDir(io, path, .{ .iterate = true }) catch return .not_synced;
defer dir.close(io);
if (!cli.isCheckoutRoot(io, dir)) {
var it = dir.iterate();
return if ((it.next(io) catch null) == null) .not_synced else .not_a_checkout;
}
var ck_buf: [checkout.max_ck_payload + 1]u8 = undefined;
const ck = (checkout.read(io, dir, &ck_buf) catch return .not_a_checkout) orelse return .not_a_checkout;
if (std.mem.eql(u8, ck.version, pin_hash)) return .at_pin;
const n = @min(ck.version.len, version_buf.len);
@memcpy(version_buf[0..n], ck.version[0..n]);
return .{ .off_pin = version_buf[0..n] };
}
fn short(hash: []const u8) []const u8 {
return hash[0..@min(hash.len, 12)];
}
fn list(arena: std.mem.Allocator, io: std.Io, root_dir: std.Io.Dir, streams: cli.Streams) cli.Exit {
const entries = readLedger(arena, io, root_dir, streams, "") catch |err| switch (err) {
error.OutOfMemory => return oom(streams),
error.InvalidLedger => return .failure,
};
if (entries.len == 0) {
streams.out.writeAll("no modules\n") catch {};
return .ok;
}
for (entries) |entry| {
var buf: [64]u8 = undefined;
streams.out.print("{s} {s} {s} ", .{ entry.path, @tagName(entry.kind), short(entry.pin) }) catch {};
if (entry.kind == .git) {
streams.out.writeAll("git commit, not checked out by zit\n") catch {};
continue;
}
switch (stateOf(io, root_dir, entry.path, entry.pin, &buf)) {
.at_pin => streams.out.writeAll("at pin\n") catch {},
.off_pin => |version| streams.out.print("off pin, at {s}\n", .{short(version)}) catch {},
.not_synced => streams.out.writeAll("not synced\n") catch {},
.not_a_checkout => streams.out.writeAll("not a zit checkout\n") catch {},
}
}
return .ok;
}
/// `zit status`'s module lines: one per zit module that is not at its pin.
/// Says nothing when there is no ledger, or it does not parse (`zit
/// modules` names that), so status never fails over a module.
pub fn reportOffPin(arena: std.mem.Allocator, io: std.Io, root_dir: std.Io.Dir, out: *std.Io.Writer) void {
const bytes = root_dir.readFileAlloc(io, modules_ledger.name, arena, .limited(1 << 24)) catch return;
var diagnostic: modules_ledger.Diagnostic = .{};
const ledger = modules_ledger.parse(arena, bytes, &diagnostic) catch return;
for (ledger.entries) |entry| {
if (entry.kind != .zit) continue;
var buf: [64]u8 = undefined;
switch (stateOf(io, root_dir, entry.path, entry.pin, &buf)) {
.at_pin => {},
.off_pin => |version| out.print("MODULE {s} at {s}, pinned {s}\n", .{ entry.path, short(version), short(entry.pin) }) catch {},
.not_synced => out.print("MODULE {s} not synced, pinned {s}\n", .{ entry.path, short(entry.pin) }) catch {},
.not_a_checkout => out.print("MODULE {s} is not a zit checkout\n", .{entry.path}) catch {},
}
}
}
/// The saved remote of the store `dir` checks out, copied into `arena`, or
/// null when there is none.
fn savedRemote(gpa: std.mem.Allocator, io: std.Io, arena: std.mem.Allocator, dir: std.Io.Dir) ?[]const u8 {
var file = cli.openStoreRead(io, dir) catch return null;
defer file.close(io);
const st = file.stat(io) catch return null;
var fs = storage.FileStorage.init(io, file, st.size);
var log = logstore.LogStore.openReadOnly(gpa, fs.storageFile()) catch return null;
defer log.deinit(gpa);
var config = (log.readConfig(gpa) catch return null) orelse return null;
defer config.deinit(gpa);
if (config.config.remote_url.len == 0) return null;
return arena.dupe(u8, config.config.remote_url) catch null;
}
/// `url` as given, or resolved against `base` when it is relative (`./`
/// or `../`), the way Git resolves a submodule url against its parent's
/// remote: each `../` drops one path segment of the base.
pub fn resolveUrl(arena: std.mem.Allocator, url: []const u8, base: ?[]const u8) error{ OutOfMemory, NoBase }![]const u8 {
if (!std.mem.startsWith(u8, url, "./") and !std.mem.startsWith(u8, url, "../")) return url;
var resolved = std.mem.trimEnd(u8, base orelse return error.NoBase, "/");
var rel = url;
// Bounded by the url's length: each pass consumes at least two bytes.
while (true) {
if (std.mem.startsWith(u8, rel, "./")) {
rel = rel[2..];
} else if (std.mem.startsWith(u8, rel, "../")) {
rel = rel[3..];
const slash = std.mem.lastIndexOfScalar(u8, resolved, '/') orelse return error.NoBase;
if (std.mem.endsWith(u8, resolved[0..slash], "/") or slash == 0) return error.NoBase; // past the host
resolved = resolved[0..slash];
} else break;
}
return std.mem.concat(arena, u8, &.{ resolved, "/", rel });
}
/// Run this zit binary with `args` in `cwd`, its output passed through.
fn runSelf(gpa: std.mem.Allocator, io: std.Io, arena: std.mem.Allocator, cwd: []const u8, args: []const []const u8) bool {
const exe = std.process.executablePathAlloc(io, gpa) catch return false;
defer gpa.free(exe);
const argv = std.mem.concat(arena, []const u8, &.{ &.{exe}, args }) catch return false;
var child = std.process.spawn(io, .{ .argv = argv, .cwd = .{ .path = cwd } }) catch return false;
const term = child.wait(io) catch return false;
return switch (term) {
.exited => |code| code == 0,
else => false,
};
}
const SyncError = error{ OutOfMemory, InvalidLedger };
/// Sync the zit modules of the checkout at `root_abs` named by `only` (all
/// when empty), then, while `depth` allows, their own. `seen` holds each
/// resolved url synced in this run, so a module reached twice syncs once.
fn syncTree(
gpa: std.mem.Allocator,
io: std.Io,
arena: std.mem.Allocator,
root_dir: std.Io.Dir,
root_abs: []const u8,
only: []const []const u8,
depth: u32,
seen: *std.StringHashMapUnmanaged(void),
streams: cli.Streams,
failures: *usize,
) SyncError!void {
std.debug.assert(depth >= 1 and depth <= max_depth);
const entries = try readLedger(arena, io, root_dir, streams, "");
for (only) |path| {
for (entries) |entry| {
if (std.mem.eql(u8, entry.path, path)) break;
} else {
streams.err.print("zit modules: no module at '{s}' in {s}\n", .{ path, modules_ledger.name }) catch {};
failures.* += 1;
}
}
var base_remote: ?[]const u8 = null;
var base_read = false;
var horizon: ?bool = null;
for (entries) |entry| {
if (only.len != 0) {
for (only) |path| {
if (std.mem.eql(u8, path, entry.path)) break;
} else continue;
}
if (entry.kind == .git) {
streams.err.print("zit modules: {s} pins a Git commit; re-import the parent with 'import-git --with-submodules' to make it a zit module\n", .{entry.path}) catch {};
failures.* += 1;
continue;
}
const raw_url = entry.url orelse {
streams.err.print("zit modules: {s} has no url to sync from\n", .{entry.path}) catch {};
failures.* += 1;
continue;
};
if (!base_read) {
base_remote = savedRemote(gpa, io, arena, root_dir);
base_read = true;
}
const url = resolveUrl(arena, raw_url, base_remote) catch |err| switch (err) {
error.OutOfMemory => return error.OutOfMemory,
error.NoBase => {
streams.err.print("zit modules: {s}: relative url '{s}' needs a saved remote to resolve against\n", .{ entry.path, raw_url }) catch {};
failures.* += 1;
continue;
},
};
const module_abs = try std.fs.path.join(arena, &.{ root_abs, entry.path });
if (seen.contains(url)) {
streams.out.print("{s}: already synced from {s} in this run\n", .{ entry.path, url }) catch {};
continue;
}
try seen.put(arena, try arena.dupe(u8, url), {});
var buf: [64]u8 = undefined;
switch (stateOf(io, root_dir, entry.path, entry.pin, &buf)) {
.at_pin => streams.out.print("{s}: at pin {s}\n", .{ entry.path, short(entry.pin) }) catch {},
.not_a_checkout => {
streams.err.print("zit modules: {s} exists and is not a zit checkout; move it aside to sync\n", .{entry.path}) catch {};
failures.* += 1;
continue;
},
.not_synced => {
if (horizon == null) horizon = isHorizon(gpa, io, root_dir);
if (std.fs.path.dirname(module_abs)) |parent| std.Io.Dir.cwd().createDirPath(io, parent) catch {};
const horizon_flag = try std.fmt.allocPrint(arena, "--horizon={s}", .{entry.pin});
const args: []const []const u8 = if (horizon.?)
&.{ "clone", horizon_flag, "--structural", url, module_abs }
else
&.{ "clone", url, module_abs };
if (!runSelf(gpa, io, arena, root_abs, args) or !runSelf(gpa, io, arena, module_abs, &.{ "update", entry.pin })) {
streams.err.print("zit modules: {s}: sync from {s} failed\n", .{ entry.path, url }) catch {};
failures.* += 1;
continue;
}
streams.out.print("{s}: cloned, at pin {s}\n", .{ entry.path, short(entry.pin) }) catch {};
},
.off_pin => {
// Update straight to the pin when it is held; pull only for one
// that is not, so an offline sync to a known pin still works.
if (!runSelf(gpa, io, arena, module_abs, &.{ "update", entry.pin })) {
if (!runSelf(gpa, io, arena, module_abs, &.{"pull"}) or !runSelf(gpa, io, arena, module_abs, &.{ "update", entry.pin })) {
streams.err.print("zit modules: {s}: cannot reach pin {s}\n", .{ entry.path, short(entry.pin) }) catch {};
failures.* += 1;
continue;
}
}
streams.out.print("{s}: updated to pin {s}\n", .{ entry.path, short(entry.pin) }) catch {};
},
}
if (depth > 1) {
var module_dir = root_dir.openDir(io, entry.path, .{}) catch continue;
defer module_dir.close(io);
try syncTree(gpa, io, arena, module_dir, module_abs, &.{}, depth - 1, seen, streams, failures);
}
}
}
fn isHorizon(gpa: std.mem.Allocator, io: std.Io, root_dir: std.Io.Dir) bool {
var store = (cli.storeDirOf(io, root_dir) catch return false) orelse return false;
defer store.close(io);
var cone = repocache.readStoreCone(gpa, io, store) orelse return false;
defer cone.deinit(gpa);
return cone.horizon != null;
}
fn pin(gpa: std.mem.Allocator, io: std.Io, arena: std.mem.Allocator, root_dir: std.Io.Dir, paths: []const []const u8, streams: cli.Streams) cli.Exit {
const existing = readLedger(arena, io, root_dir, streams, "") catch |err| switch (err) {
error.OutOfMemory => return oom(streams),
error.InvalidLedger => return .failure,
};
const had_ledger = existing.len != 0;
var entries: std.ArrayList(modules_ledger.Entry) = .empty;
entries.appendSlice(arena, existing) catch return oom(streams);
for (paths) |path| {
var dir = root_dir.openDir(io, path, .{}) catch {
streams.err.print("zit modules: no checkout at '{s}'\n", .{path}) catch {};
return .failure;
};
defer dir.close(io);
if (!cli.isCheckoutRoot(io, dir)) {
streams.err.print("zit modules: '{s}' is not a zit checkout\n", .{path}) catch {};
return .failure;
}
var ck_buf: [checkout.max_ck_payload + 1]u8 = undefined;
const ck = (checkout.read(io, dir, &ck_buf) catch null) orelse {
streams.err.print("zit modules: cannot read the checkout at '{s}'\n", .{path}) catch {};
return .failure;
};
if (!modules_ledger.isFullHash(ck.version)) {
streams.err.print("zit modules: '{s}' has no check-in to pin\n", .{path}) catch {};
return .failure;
}
const version = arena.dupe(u8, ck.version) catch return oom(streams);
const index = for (entries.items, 0..) |entry, i| {
if (std.mem.eql(u8, entry.path, path)) break i;
} else null;
if (index) |i| {
const old = entries.items[i];
// A git line has no zit url; the checkout's own remote is the one.
const url = if (old.kind == .zit) old.url else savedRemote(gpa, io, arena, dir);
entries.items[i] = .{ .path = old.path, .kind = .zit, .url = url, .pin = version };
} else {
entries.append(arena, .{ .path = arena.dupe(u8, path) catch return oom(streams), .kind = .zit, .url = savedRemote(gpa, io, arena, dir), .pin = version }) catch return oom(streams);
}
streams.out.print("{s}: pinned at {s}\n", .{ path, short(version) }) catch {};
}
modules_ledger.sort(entries.items);
const bytes = modules_ledger.format(arena, entries.items) catch {
streams.err.writeAll("zit modules: a module path or url cannot be written to the ledger\n") catch {};
return .failure;
};
root_dir.writeFile(io, .{ .sub_path = modules_ledger.name, .data = bytes }) catch |err| {
streams.err.print("zit modules: cannot write {s}: {s}\n", .{ modules_ledger.name, @errorName(err) }) catch {};
return .failure;
};
if (!had_ledger) streams.out.print("track it with 'zit add {s}'\n", .{modules_ledger.name}) catch {};
return .ok;
}
test "modules: a relative url resolves against the parent's remote as Git resolves it" {
var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator);
defer arena_state.deinit();
const a = arena_state.allocator();
try std.testing.expectEqualStrings("https://h/a/lib", try resolveUrl(a, "../lib", "https://h/a/parent"));
try std.testing.expectEqualStrings("https://h/a/parent/lib", try resolveUrl(a, "./lib", "https://h/a/parent/"));
try std.testing.expectEqualStrings("https://h/x", try resolveUrl(a, "../../x", "https://h/a/parent"));
try std.testing.expectEqualStrings("https://other/lib", try resolveUrl(a, "https://other/lib", null));
try std.testing.expectError(error.NoBase, resolveUrl(a, "../lib", null));
try std.testing.expectError(error.NoBase, resolveUrl(a, "../../../x", "https://h/a"));
}