//! `.zit-modules`, the tracked file that pins a check-in's submodules.
//!
//! Git records a submodule as a gitlink: a tree entry of mode 160000 naming a
//! commit in another repository. Fossil's artifact format has no such entry,
//! so `fossil import --git` drops the pin, and so did zit. The ledger keeps
//! it as file content instead, which every Fossil peer stores and syncs as a
//! plain file and which leaves the artifact format untouched. `import-git`
//! writes it from the gitlinks it meets, and `export-git` turns it back into
//! gitlinks rather than copying it, so a Git round trip gives the same trees.
//!
//! One module per line, sorted by path in byte order, four fields separated by
//! one space:
//!
//! path kind url pin
//!
//! - `path` is the module's path in the tree, spelled with the F-card
//! filename escapes (`\s` for a space), so it is one token.
//! - `kind` is `git` (the pin is a Git commit) or `zit` (a zit check-in).
//! - `url` is where the module lives, escaped the same way, or `-` when no
//! `.gitmodules` entry names one. A relative URL resolves against the
//! superproject's remote, as Git resolves it.
//! - `pin` is a full lowercase hash, 40 or 64 hex digits; never a prefix or
//! a branch name.
const std = @import("std");
const artifact = @import("artifact.zig");
/// The ledger's path, at the root of the tree.
pub const name = ".zit-modules";
/// Longest field after escaping; a path is bounded by the F-card filename
/// limit long before this.
pub const max_field_bytes: usize = 8192;
/// Most modules one ledger holds.
pub const max_entries: usize = 65536;
pub const Kind = enum { git, zit };
/// One module. Strings are decoded (no escapes) and owned by the `Ledger`
/// that holds the entry, or by the caller when built by hand.
pub const Entry = struct {
path: []const u8,
kind: Kind,
/// Null when unknown, written `-`.
url: ?[]const u8,
pin: []const u8,
};
pub const Ledger = struct {
entries: []Entry,
arena: std.heap.ArenaAllocator,
pub fn deinit(self: *Ledger) void {
self.arena.deinit();
self.* = undefined;
}
/// The entry for `path`, or null.
pub fn find(self: *const Ledger, path: []const u8) ?Entry {
for (self.entries) |entry| if (std.mem.eql(u8, entry.path, path)) return entry;
return null;
}
};
pub const ParseError = error{ OutOfMemory, InvalidLedger };
/// Why a ledger was refused, and on which 1-based line.
pub const Diagnostic = struct {
line: usize = 0,
reason: []const u8 = "",
};
/// Parse ledger bytes. Every rule above is checked, so a hand-edited file
/// that breaks one is refused with its line rather than read loosely.
pub fn parse(gpa: std.mem.Allocator, bytes: []const u8, diagnostic: *Diagnostic) ParseError!Ledger {
var arena = std.heap.ArenaAllocator.init(gpa);
errdefer arena.deinit();
const a = arena.allocator();
var entries: std.ArrayList(Entry) = .empty;
if (bytes.len != 0 and bytes[bytes.len - 1] != '\n') return refuse(diagnostic, std.mem.count(u8, bytes, "\n") + 1, "the last line has no newline");
var lines = std.mem.splitScalar(u8, bytes, '\n');
var number: usize = 0;
while (lines.next()) |line| {
number += 1;
if (line.len == 0) {
if (lines.peek() == null) break; // the split's empty tail after the final newline
return refuse(diagnostic, number, "an empty line");
}
if (entries.items.len == max_entries) return refuse(diagnostic, number, "more modules than a ledger holds");
var fields: [4][]const u8 = undefined;
var it = std.mem.splitScalar(u8, line, ' ');
var count: usize = 0;
while (it.next()) |field| : (count += 1) {
if (count == fields.len) return refuse(diagnostic, number, "more than four fields");
if (field.len == 0 or field.len > max_field_bytes) return refuse(diagnostic, number, "an empty or oversized field");
fields[count] = field;
}
if (count != fields.len) return refuse(diagnostic, number, "fewer than four fields");
const kind = std.meta.stringToEnum(Kind, fields[1]) orelse return refuse(diagnostic, number, "a kind that is neither git nor zit");
if (!isFullHash(fields[3])) return refuse(diagnostic, number, "a pin that is not a full lowercase hash");
const path = try decode(a, fields[0]);
if (!validPath(path)) return refuse(diagnostic, number, "a path a tree cannot hold");
if (entries.items.len != 0 and !lessThan(entries.items[entries.items.len - 1].path, path)) {
return refuse(diagnostic, number, "a path out of order or repeated");
}
const url: ?[]const u8 = if (std.mem.eql(u8, fields[2], "-")) null else try decode(a, fields[2]);
try entries.append(a, .{ .path = path, .kind = kind, .url = url, .pin = try a.dupe(u8, fields[3]) });
}
return .{ .entries = try entries.toOwnedSlice(a), .arena = arena };
}
/// The ledger bytes for `entries`, which the caller has sorted by path
/// (`sort`). Empty for no entries: a tree with no modules holds no ledger.
/// Refuses what `parse` would refuse, so nothing written fails to read back.
pub fn format(gpa: std.mem.Allocator, entries: []const Entry) error{ OutOfMemory, InvalidLedger }![]u8 {
var out: std.ArrayList(u8) = .empty;
errdefer out.deinit(gpa);
var buf: [2 * max_field_bytes]u8 = undefined;
for (entries, 0..) |entry, i| {
if (entries.len > max_entries) return error.InvalidLedger;
if (entry.path.len == 0 or entry.path.len > max_field_bytes or !validPath(entry.path)) return error.InvalidLedger;
if (!isFullHash(entry.pin)) return error.InvalidLedger;
if (i != 0 and !lessThan(entries[i - 1].path, entry.path)) return error.InvalidLedger;
try out.appendSlice(gpa, artifact.encodeEscapes(entry.path, &buf));
try out.append(gpa, ' ');
try out.appendSlice(gpa, @tagName(entry.kind));
try out.append(gpa, ' ');
if (entry.url) |url| {
if (url.len == 0 or url.len > max_field_bytes) return error.InvalidLedger;
try out.appendSlice(gpa, artifact.encodeEscapes(url, &buf));
} else try out.append(gpa, '-');
try out.append(gpa, ' ');
try out.appendSlice(gpa, entry.pin);
try out.append(gpa, '\n');
}
return out.toOwnedSlice(gpa);
}
/// Sort entries by path, the order `format` writes and `parse` requires.
pub fn sort(entries: []Entry) void {
std.mem.sort(Entry, entries, {}, struct {
fn less(_: void, x: Entry, y: Entry) bool {
return lessThan(x.path, y.path);
}
}.less);
}
/// Byte order, the order F cards are sorted in.
fn lessThan(x: []const u8, y: []const u8) bool {
return std.mem.order(u8, x, y) == .lt;
}
/// A full lowercase hash: 40 hex digits (SHA-1) or 64 (SHA-256).
pub fn isFullHash(text: []const u8) bool {
if (text.len != 40 and text.len != 64) return false;
for (text) |c| switch (c) {
'0'...'9', 'a'...'f' => {},
else => return false,
};
return true;
}
/// A relative path with no empty, `.` or `..` component, as a tree entry is.
fn validPath(path: []const u8) bool {
if (path.len == 0 or path[0] == '/' or path[path.len - 1] == '/') return false;
var parts = std.mem.splitScalar(u8, path, '/');
while (parts.next()) |part| {
if (part.len == 0 or std.mem.eql(u8, part, ".") or std.mem.eql(u8, part, "..")) return false;
}
return std.mem.indexOfScalar(u8, path, 0) == null;
}
fn decode(a: std.mem.Allocator, field: []const u8) error{OutOfMemory}![]const u8 {
const out = try a.alloc(u8, field.len);
return artifact.decodeEscapes(field, out);
}
fn refuse(diagnostic: *Diagnostic, line: usize, reason: []const u8) error{InvalidLedger} {
diagnostic.* = .{ .line = line, .reason = reason };
return error.InvalidLedger;
}
// ---------------------------------------------------------------------------
// .gitmodules
// ---------------------------------------------------------------------------
/// One `[submodule]` section's path and url, as Git's `.gitmodules` gives
/// them. Strings borrow the parsed text or `UrlMap.arena`.
pub const UrlMap = struct {
paths: std.StringHashMapUnmanaged([]const u8) = .empty,
arena: std.heap.ArenaAllocator,
pub fn deinit(self: *UrlMap) void {
self.arena.deinit();
self.* = undefined;
}
pub fn urlOf(self: *const UrlMap, path: []const u8) ?[]const u8 {
return self.paths.get(path);
}
};
/// Read the path-to-url pairs of a `.gitmodules` file. Git-config syntax,
/// only as much of it as a `.gitmodules` uses: `[submodule "NAME"]`
/// sections, `key = value` lines, `#` and `;` comments, and double-quoted
/// values. A section missing either key, or a line this cannot read, adds
/// nothing: an unreadable entry leaves that module's url unknown, which the
/// ledger records as `-`, and never fails an import.
pub fn parseGitmodules(gpa: std.mem.Allocator, text: []const u8) error{OutOfMemory}!UrlMap {
var map: UrlMap = .{ .arena = std.heap.ArenaAllocator.init(gpa) };
errdefer map.deinit();
const a = map.arena.allocator();
var in_submodule = false;
var path: ?[]const u8 = null;
var url: ?[]const u8 = null;
var lines = std.mem.splitScalar(u8, text, '\n');
while (lines.next()) |raw| {
const line = std.mem.trim(u8, raw, " \t\r");
if (line.len == 0 or line[0] == '#' or line[0] == ';') continue;
if (line[0] == '[') {
try finishSection(a, &map, in_submodule, path, url);
path = null;
url = null;
in_submodule = std.mem.startsWith(u8, line, "[submodule");
continue;
}
if (!in_submodule) continue;
const eq = std.mem.indexOfScalar(u8, line, '=') orelse continue;
const key = std.mem.trim(u8, line[0..eq], " \t");
const value = unquote(std.mem.trim(u8, line[eq + 1 ..], " \t"));
if (std.ascii.eqlIgnoreCase(key, "path")) path = value;
if (std.ascii.eqlIgnoreCase(key, "url")) url = value;
}
try finishSection(a, &map, in_submodule, path, url);
return map;
}
fn finishSection(a: std.mem.Allocator, map: *UrlMap, in_submodule: bool, path: ?[]const u8, url: ?[]const u8) error{OutOfMemory}!void {
if (!in_submodule) return;
const p = path orelse return;
const u = url orelse return;
if (p.len == 0 or u.len == 0 or u.len > max_field_bytes) return;
try map.paths.put(a, p, u);
}
fn unquote(value: []const u8) []const u8 {
if (value.len >= 2 and value[0] == '"' and value[value.len - 1] == '"') return value[1 .. value.len - 1];
return value;
}
// ---------------------------------------------------------------------------
// tests
// ---------------------------------------------------------------------------
const testing = std.testing;
const pin_a = "1111111111111111111111111111111111111111";
const pin_b = "2222222222222222222222222222222222222222222222222222222222222222";
test "modules ledger: format and parse round-trip, with a space in a path and an unknown url" {
var entries = [_]Entry{
.{ .path = "vendor/my lib", .kind = .git, .url = null, .pin = pin_b },
.{ .path = "lib", .kind = .git, .url = "https://example.test/lib.git", .pin = pin_a },
};
sort(&entries);
const bytes = try format(testing.allocator, &entries);
defer testing.allocator.free(bytes);
try testing.expectEqualStrings("lib git https://example.test/lib.git " ++ pin_a ++ "\n" ++
"vendor/my\\slib git - " ++ pin_b ++ "\n", bytes);
var diagnostic: Diagnostic = .{};
var ledger = try parse(testing.allocator, bytes, &diagnostic);
defer ledger.deinit();
try testing.expectEqual(@as(usize, 2), ledger.entries.len);
try testing.expectEqualStrings("vendor/my lib", ledger.entries[1].path);
try testing.expect(ledger.entries[1].url == null);
try testing.expectEqualStrings("https://example.test/lib.git", ledger.find("lib").?.url.?);
const empty = try format(testing.allocator, &.{});
defer testing.allocator.free(empty);
try testing.expectEqualStrings("", empty);
}
test "modules ledger: a broken line is refused with its line number" {
const cases = [_]struct { bytes: []const u8, line: usize }{
.{ .bytes = "lib git - 1111\n", .line = 1 }, // a short pin
.{ .bytes = "a git - " ++ pin_a ++ "\nlib git - main\n", .line = 2 }, // a branch name
.{ .bytes = "lib git - " ++ pin_a ++ " extra\n", .line = 1 }, // a fifth field
.{ .bytes = "lib svn - " ++ pin_a ++ "\n", .line = 1 }, // an unknown kind
.{ .bytes = "b git - " ++ pin_a ++ "\na git - " ++ pin_a ++ "\n", .line = 2 }, // out of order
.{ .bytes = "a git - " ++ pin_a ++ "\na git - " ++ pin_a ++ "\n", .line = 2 }, // repeated
.{ .bytes = "../a git - " ++ pin_a ++ "\n", .line = 1 }, // a path outside the tree
.{ .bytes = "lib git - " ++ pin_a, .line = 1 }, // no final newline
.{ .bytes = "a git - " ++ pin_a ++ "\n\nb git - " ++ pin_a ++ "\n", .line = 2 }, // an empty line
.{ .bytes = "lib git - " ++ "1111111111111111111111111111111111111ABC" ++ "\n", .line = 1 }, // uppercase
};
for (cases) |case| {
var diagnostic: Diagnostic = .{};
try testing.expectError(error.InvalidLedger, parse(testing.allocator, case.bytes, &diagnostic));
try testing.expectEqual(case.line, diagnostic.line);
try testing.expect(diagnostic.reason.len != 0);
}
}
test "modules ledger: .gitmodules gives each section's path and url" {
const text = "# comment\n[core]\n\tpath = not-a-module\n" ++
"[submodule \"lib\"]\n\tpath = lib\n\turl = https://example.test/lib.git\n" ++
"[submodule \"spaced\"]\n\turl = \"../my lib.git\"\n\tpath = vendor/my lib\n" ++
"[submodule \"no-url\"]\n\tpath = bare\n";
var map = try parseGitmodules(testing.allocator, text);
defer map.deinit();
try testing.expectEqualStrings("https://example.test/lib.git", map.urlOf("lib").?);
try testing.expectEqualStrings("../my lib.git", map.urlOf("vendor/my lib").?);
try testing.expect(map.urlOf("bare") == null);
try testing.expect(map.urlOf("not-a-module") == null);
}