//! `zit crosslink`: stream the derived per-check-in records.
const std = @import("std");
const zit = @import("zit");
const cli = @import("zit").cli;
const logstore = @import("zit").logstore;
const Repository = @import("zit").Repository;
const artifact = zit.artifact;
const diagnostics = @import("../diagnostics.zig");
const help_pages = @import("../help.zig");
const stateOom = diagnostics.stateOom;
const repoOpenFail = diagnostics.repoOpenFail;
const openRepo = diagnostics.openRepo;
/// Set from `$ZIT_LINES_WHOLE_TREE` at the same seam: `crosslink --lines`
/// reads each old side from the parent's whole file set rather than by
/// name, so the differential test can prove the two count the same.
pub var lines_whole_tree: bool = false;
// ---------------------------------------------------------------------------
// zit crosslink: stream the derived per-check-in records.
// ---------------------------------------------------------------------------
/// The stream-format version `runCrosslink` emits as its first line. Bump it
/// when a line's meaning changes; consumers refuse versions they do not know.
const crosslink_stream_version = "crosslink 1";
pub fn runCrosslink(
gpa: std.mem.Allocator,
io: std.Io,
arena: std.mem.Allocator,
verb_args: []const []const u8,
streams: cli.Streams,
amb: cli.Ambient,
) cli.Exit {
var bypass = false;
var lines = false;
for (verb_args) |tok| {
if (std.mem.eql(u8, tok, "-h") or std.mem.eql(u8, tok, "--help")) {
streams.out.writeAll(help_pages.crosslink_help) catch {};
return .ok;
} else if (std.mem.eql(u8, tok, "--no-cache")) {
bypass = true;
} else if (std.mem.eql(u8, tok, "--lines")) {
lines = true;
} else if (tok.len > 1 and tok[0] == '-') {
return cli.usageError(streams, "crosslink", .unknown_option, tok);
} else {
return cli.usageError(streams, "crosslink", .extra_argument, tok);
}
}
// The open heals a stale cache (the extension path), so the
// cache-served branch below almost always finds a body covering the whole
// log. `--no-cache` opens with the bypass, so nothing below can be
// answered by a sidecar โ the audit path this verb's byte-equality is
// pinned against.
var repo: Repository = undefined;
var root_dir: std.Io.Dir = undefined;
const rc = openRepo(gpa, io, arena, streams, "crosslink", amb.repo_override, bypass, !bypass, &repo, &root_dir);
if (rc != .ok) return rc;
defer root_dir.close(io);
defer repo.deinit(gpa);
const log = repo.logStore(gpa) catch |e| return repoOpenFail(streams, "crosslink", e);
// Serve from the cache's record body when it provably covers the whole
// log โ same key + shun-fingerprint discipline as the extension path โ
// else derive the records fresh from the log. Both paths emit through one
// encoder, and the e2e pins them byte-identical.
var derived: std.ArrayList(u8) = .empty;
defer derived.deinit(gpa);
var raw: ?zit.repocache.RawCache = null;
defer if (raw) |*rw| rw.deinit();
var body: []const u8 = undefined;
served_from_cache: {
if (!bypass) {
if (zit.repocache.loadForExtension(io, repo.root_dir)) |raw_const| {
raw = raw_const;
const key = log.logKey() catch break :served_from_cache;
// A body an earlier release wrote is served only once the
// open has upgraded it; one it could not rewrite is derived.
if (!raw.?.legacy and raw.?.shun_fp == log.shunFingerprint() and std.meta.eql(raw.?.key, key)) {
body = raw.?.body;
break :served_from_cache;
}
raw.?.deinit();
raw = null;
}
}
zit.repoview.captureRecords(gpa, log, 0, std.math.maxInt(u64), &derived) catch |e| switch (e) {
error.OutOfMemory => return stateOom(streams, "crosslink"),
else => {
streams.err.writeAll("zit crosslink: the store could not be read\n") catch {};
return .failure;
},
};
body = derived.items;
}
if (!lines) return crosslinkEmit(gpa, streams, body, null);
// Line counts come from their own sidecar, never from the record body:
// counting reads every changed blob, and the body is built by any verb's
// first open. Only `--lines` pays, and only once per check-in.
// `--no-cache` counts everything afresh and stores nothing.
const shun_fp = log.shunFingerprint();
var counter: LineCounter = .{ .repo = &repo, .shun_fp = shun_fp, .log_entries = log.entryCount(), .io = io, .persist = !bypass };
defer counter.deinit(gpa);
if (!bypass) {
if (zit.repocache.tryLoadLineStats(gpa, io, repo.root_dir)) |stored| {
counter.stored = stored;
counter.reader = zit.linestats.Reader.open(stored);
if (counter.reader) |r| if (r.shunFp() != shun_fp) {
counter.shun_moved = true;
const keep = counter.markShunned(gpa, arena, log, body) catch return stateOom(streams, "crosslink");
if (!keep) counter.reader = null;
};
}
}
const rc_emit = crosslinkEmit(gpa, streams, body, &counter);
if (!bypass and rc_emit == .ok) {
if (counter.writer) |w| {
zit.repocache.storeLineStats(io, repo.root_dir, w.out.items);
} else if ((counter.retried or counter.shun_moved) and counter.reader != null) {
// Every entry was met and nothing needed rewriting: record the
// shun table and the log the entries now stand for, so the next
// run neither re-checks the shun nor retries the placeholders
// until either moves again.
if (counter.reader.?.restamped(gpa, shun_fp, counter.log_entries)) |bytes| {
defer gpa.free(bytes);
zit.repocache.storeLineStats(io, repo.root_dir, bytes);
} else |_| {}
}
}
return rc_emit;
}
/// Each changed file's added and removed lines against the primary parent,
/// counted the way `zit diff --numstat --checkin` counts them: the parent's
/// blob at the same path against the check-in's. Served from the stored
/// counts when an entry lines up with the record, counted otherwise.
///
/// While every record finds its stored entry next in line, nothing is
/// written. The first record that does not (a new check-in, a placeholder
/// that now counts, an entry out of order) starts `writer` with the stored
/// entries before it, and from then on every record is appended; the result
/// is stored when the stream ends.
const LineCounter = struct {
repo: *Repository,
shun_fp: u32,
/// The log's entry count now, stamped on what is stored.
log_entries: u64,
io: std.Io,
/// A placeholder was counted again because the log grew.
retried: bool = false,
/// The shun table moved since the counts were stored: an entry for a
/// check-in that touches a path in `shunned_paths` is counted again.
shun_moved: bool = false,
/// Every path a shunned blob was ever written to. A shunned blob reaches
/// a count as the new side of the check-in that wrote it, or as the old
/// side of a later one that changed the same path; both touch the path.
shunned_paths: std.StringHashMapUnmanaged(void) = .empty,
/// Store what was counted (false under `--no-cache`).
persist: bool,
/// Check-ins counted since the writer was last stored. A first run over
/// a long history stores its prefix every `flush_every`, so a run cut
/// short (a timeout, a restart) is resumed rather than repeated.
unflushed: usize = 0,
stored: ?[]u8 = null,
reader: ?zit.linestats.Reader = null,
writer: ?zit.linestats.Writer = null,
named: Repository.NamedMemo = .{},
/// Every blob a count reads, whole, under the cache's byte bound -- not
/// only the delta bases it keeps by default. A file's old side is
/// usually the new side of an earlier check-in on its line, read moments
/// before.
plain: logstore.PlaintextCache = .{ .memoize_undeltaed = true },
scratch: std.ArrayList(zit.linestats.Count) = .empty,
const Count = zit.linestats.Count;
const flush_every: usize = 4096;
fn deinit(self: *LineCounter, gpa: std.mem.Allocator) void {
if (self.reader) |*r| r.deinit(gpa);
if (self.stored) |b| gpa.free(b);
if (self.writer) |*w| w.deinit(gpa);
self.named.deinit(gpa);
self.shunned_paths.deinit(gpa);
self.plain.deinit(gpa);
self.scratch.deinit(gpa);
}
/// The counts for `rec`'s files, one per file in order. The slice is
/// valid until the next call.
fn forRecord(self: *LineCounter, gpa: std.mem.Allocator, rec: *const zit.repoview.DecodedRecord) error{OutOfMemory}![]const Count {
// Where this record's entry starts if the stored entries are still
// being met in order: everything before it is what the writer would
// hold, should it have to start here.
const in_order_at: ?usize = if (self.reader) |r| (if (r.inOrder()) r.at else null) else null;
const found: zit.linestats.Reader.Found = if (self.reader) |*r| try r.find(gpa, rec.hash, &self.scratch) else .miss;
// A placeholder against the log it was counted on stays one: the
// content it lacked has not arrived.
if (found == .recount and self.reader.?.logEntries() == self.log_entries) {
self.scratch.clearRetainingCapacity();
try self.scratch.appendNTimes(gpa, Count.unknown, rec.files.items.len);
if (self.writer == null and !self.reader.?.inOrder()) try self.startWriter(gpa, in_order_at);
if (self.writer) |*w| try w.append(gpa, rec.hash, self.scratch.items);
return self.scratch.items;
}
if (found == .recount) self.retried = true;
const usable = found == .hit and self.scratch.items.len == rec.files.items.len and !self.touchesShunned(rec);
if (usable) {
if (self.writer == null and !self.reader.?.inOrder()) try self.startWriter(gpa, in_order_at);
if (self.writer) |*w| try w.append(gpa, rec.hash, self.scratch.items);
return self.scratch.items;
}
self.scratch.clearRetainingCapacity();
try self.scratch.appendNTimes(gpa, Count.unknown, rec.files.items.len);
try self.count(gpa, rec, self.scratch.items);
const complete = for (self.scratch.items) |c| {
if (c.eql(Count.unknown)) break false;
} else true;
// A placeholder that still cannot be counted, met in order, changes
// nothing that is stored.
const unchanged = found == .recount and !complete and self.reader.?.inOrder();
if (self.writer == null and !unchanged) try self.startWriter(gpa, in_order_at);
if (self.writer) |*w| {
try w.append(gpa, rec.hash, self.scratch.items);
self.unflushed += 1;
if (self.persist and self.unflushed >= flush_every) {
zit.repocache.storeLineStats(self.io, self.repo.root_dir, w.out.items);
self.unflushed = 0;
}
}
return self.scratch.items;
}
/// Collect `shunned_paths` from `body`. False when the stored counts
/// must all go: a check-in whose changed files are unknown may have
/// written a shunned blob to a path nothing else records.
fn markShunned(self: *LineCounter, gpa: std.mem.Allocator, arena: std.mem.Allocator, log: *logstore.LogStore, body: []const u8) error{OutOfMemory}!bool {
if (log.shunList().len == 0) return true;
var it = zit.repoview.RecordIter.init(body);
while (it.next(gpa) catch return false) |rec_const| {
var rec = rec_const;
defer rec.deinit(gpa);
if (rec.kind != .manifest) continue;
if (!rec.files_known) return false;
for (rec.files.items) |f| {
const h = f.hash orelse continue;
if (!isShunnedHex(log, h)) continue;
if (!self.shunned_paths.contains(f.path)) try self.shunned_paths.put(gpa, try arena.dupe(u8, f.path), {});
}
}
return true;
}
fn touchesShunned(self: *const LineCounter, rec: *const zit.repoview.DecodedRecord) bool {
if (!self.shun_moved or self.shunned_paths.count() == 0) return false;
for (rec.files.items) |f| {
if (self.shunned_paths.contains(f.path)) return true;
}
return false;
}
fn isShunnedHex(log: *logstore.LogStore, hex: []const u8) bool {
var buf: [logstore.max_hash_len]u8 = undefined;
if (hex.len % 2 != 0 or hex.len / 2 > buf.len) return false;
const digest = std.fmt.hexToBytes(buf[0 .. hex.len / 2], hex) catch return false;
return logstore.isCompleteDigest(digest.len) and log.isShunned(digest);
}
/// Start the writer with the stored entries before `prefix_end` (none
/// without a stored blob or once the order was lost; then every entry
/// before this record was appended as it was met, so there is none).
fn startWriter(self: *LineCounter, gpa: std.mem.Allocator, prefix_end: ?usize) error{OutOfMemory}!void {
var w = try zit.linestats.Writer.init(gpa, self.shun_fp, self.log_entries);
errdefer w.deinit(gpa);
if (prefix_end) |end| try w.out.appendSlice(gpa, self.stored.?[zit.linestats.header_len..end]);
self.writer = w;
}
/// Fill `out` (pre-set to unknown) for every file both of whose sides
/// can be read.
fn count(self: *LineCounter, gpa: std.mem.Allocator, rec: *const zit.repoview.DecodedRecord, out: []Count) error{OutOfMemory}!void {
const log = self.repo.logStore(gpa) catch |e| switch (e) {
error.OutOfMemory => return error.OutOfMemory,
else => return,
};
// The parent-side blob of each path, null where the parent lacks it.
var arena_state = std.heap.ArenaAllocator.init(gpa);
defer arena_state.deinit();
const arena = arena_state.allocator();
const olds = try arena.alloc(Repository.NamedHash, rec.files.items.len);
for (rec.files.items, olds) |f, *o| o.* = .{ .name = f.path };
if (rec.parents.items.len != 0) {
const prid = self.repo.lookup(rec.parents.items[0]) orelse return;
const found = (if (lines_whole_tree) self.repo.hashesNamedWhole(gpa, arena, prid, olds, &self.named) else self.repo.hashesNamed(gpa, arena, prid, olds, &self.named, &self.plain)) catch |e| switch (e) {
error.OutOfMemory => return error.OutOfMemory,
else => return,
};
if (!found) return;
}
for (rec.files.items, olds, out) |f, o, *slot| {
const old_hash = o.hash;
if (old_hash != null and f.hash != null and std.mem.eql(u8, old_hash.?, f.hash.?)) {
slot.* = Count.text(0, 0); // a mode-only change: same blob both sides
continue;
}
const old = (try self.side(gpa, log, old_hash)) orelse continue;
defer gpa.free(old);
const new = (try self.side(gpa, log, if (f.disp == .deleted) null else f.hash)) orelse continue;
defer gpa.free(new);
if (zit.textdiff.isBinary(old) or zit.textdiff.isBinary(new)) {
slot.* = Count.binary;
} else {
const lc = try zit.textdiff.lineCounts(gpa, old, new, .{});
slot.* = Count.text(lc.added, lc.removed);
}
}
}
/// One side's bytes: empty for an absent side, null when the store
/// cannot produce the blob. Read straight from the log the way the derive
/// reads manifests: each frame is CRC-checked, and re-hashing every blob
/// of a history to name it again was most of a first run. Caller frees.
fn side(self: *LineCounter, gpa: std.mem.Allocator, log: *logstore.LogStore, hash: ?[]const u8) error{OutOfMemory}!?[]u8 {
const h = hash orelse return try gpa.alloc(u8, 0);
var digest_buf: [logstore.max_hash_len]u8 = undefined;
if (h.len % 2 != 0 or h.len / 2 > digest_buf.len) return null;
const digest = std.fmt.hexToBytes(digest_buf[0 .. h.len / 2], h) catch return null;
const rid = log.lookup(digest) orelse return null;
return log.contentCached(gpa, rid, &self.plain) catch |e| switch (e) {
error.OutOfMemory => return error.OutOfMemory,
else => return null,
};
}
};
/// Emit one record body as the versioned text stream. Free-text tokens (user,
/// tag names and values, paths) are card-escaped exactly as an artifact would
/// carry them, so every emitted line is newline-safe and a consumer reuses
/// the artifact-format decoder it already has. The comment is length-framed
/// raw bytes, the one field escaping cannot serve (it is free prose).
fn crosslinkEmit(gpa: std.mem.Allocator, streams: cli.Streams, body: []const u8, counter: ?*LineCounter) cli.Exit {
var scratch: std.ArrayList(u8) = .empty;
defer scratch.deinit(gpa);
var lines_buf: [lines_line_max]u8 = undefined;
streams.out.print("{s}\n", .{crosslink_stream_version}) catch return .failure;
var it = zit.repoview.RecordIter.init(body);
while (it.next(gpa) catch {
streams.err.writeAll("zit crosslink: the record body is corrupt; re-run with --no-cache\n") catch {};
return .failure;
}) |rec_const| {
var rec = rec_const;
defer rec.deinit(gpa);
if (rec.kind != .manifest) continue; // control artifacts: not check-ins
streams.out.print("checkin {s} {d}", .{ rec.hash, rec.mtime }) catch return .failure;
for (rec.parents.items) |parent| streams.out.print(" {s}", .{parent}) catch return .failure;
streams.out.writeAll("\n") catch return .failure;
if (rec.user.len != 0) {
streams.out.print("user {s}\n", .{escaped(gpa, &scratch, rec.user) catch return .failure}) catch return .failure;
}
streams.out.print("comment {d}\n", .{rec.comment.len}) catch return .failure;
streams.out.writeAll(rec.comment) catch return .failure;
streams.out.writeAll("\n") catch return .failure;
for (rec.tags.items) |t| {
const sigil: u8 = switch (t.type) {
.singleton => '+',
.cancel => '-',
.propagate => '*',
};
streams.out.print("tag {c}{s}", .{ sigil, escaped(gpa, &scratch, t.name) catch return .failure }) catch return .failure;
if (t.target) |target| {
streams.out.print(" {s}", .{target}) catch return .failure;
} else {
streams.out.writeAll(" *") catch return .failure;
}
if (t.value) |v| {
streams.out.print(" {s}", .{escaped(gpa, &scratch, v) catch return .failure}) catch return .failure;
}
streams.out.writeAll("\n") catch return .failure;
}
for (rec.picks.items) |pk| {
const sign: u8 = switch (pk.sign) {
.cherrypick => '+',
.backout => '-',
};
streams.out.print("cherrypick {c}{s}\n", .{ sign, pk.target }) catch return .failure;
}
if (rec.files_known) {
// Without `--lines` nothing is counted and no `lines` line is printed.
const counts: ?[]const LineCounter.Count = if (counter) |lc| (lc.forRecord(gpa, &rec) catch return stateOom(streams, "crosslink")) else null;
for (rec.files.items, 0..) |f, i| {
streams.out.print("file {c} {s}\n", .{ @backingInt(f.disp), escaped(gpa, &scratch, f.path) catch return .failure }) catch return .failure;
if (counts) |cs| {
const c = cs[i];
if (c.eql(LineCounter.Count.binary)) {
streams.out.writeAll("lines - -\n") catch return .failure;
} else if (!c.eql(LineCounter.Count.unknown)) {
streams.out.writeAll(linesLine(&lines_buf, c)) catch return .failure;
}
}
// The rename the check-in recorded, after the file it names,
// so a consumer that knows only A/D lines reads what it
// always read and one that knows this line carries the
// file's identity across it.
if (f.prior) |old| {
streams.out.print("rename {s}", .{escaped(gpa, &scratch, old) catch return .failure}) catch return .failure;
streams.out.print(" {s}\n", .{escaped(gpa, &scratch, f.path) catch return .failure}) catch return .failure;
}
}
} else {
streams.out.writeAll("files unknown\n") catch return .failure;
}
streams.out.writeAll("end\n") catch return .failure;
}
return .ok;
}
const lines_line_max = "lines ".len + 10 + 1 + 10 + 1;
/// `lines ADDED REMOVED\n` for a text count, formatted by hand: a long
/// history prints one per changed file, and the general formatter was most
/// of what printing them cost.
fn linesLine(buf: *[lines_line_max]u8, c: zit.linestats.Count) []const u8 {
var n: usize = 0;
@memcpy(buf[0.."lines ".len], "lines ");
n += "lines ".len;
n += decimal(buf[n..], c.added);
buf[n] = ' ';
n += 1;
n += decimal(buf[n..], c.removed);
buf[n] = '\n';
return buf[0 .. n + 1];
}
/// Write `v` in decimal at the start of `out`; returns the digit count.
fn decimal(out: []u8, v: u32) usize {
var tmp: [10]u8 = undefined;
var i: usize = tmp.len;
var x = v;
while (true) {
i -= 1;
tmp[i] = '0' + @as(u8, @intCast(x % 10));
x /= 10;
if (x == 0) break;
}
const len = tmp.len - i;
@memcpy(out[0..len], tmp[i..]);
return len;
}
test "linesLine formats like the general formatter" {
var buf: [lines_line_max]u8 = undefined;
for ([_][2]u32{ .{ 0, 0 }, .{ 12, 3 }, .{ 9, 10 }, .{ std.math.maxInt(u32) - 2, 1 } }) |p| {
var want: [lines_line_max]u8 = undefined;
const w = try std.fmt.bufPrint(&want, "lines {d} {d}\n", .{ p[0], p[1] });
try std.testing.expectEqualStrings(w, linesLine(&buf, .{ .added = p[0], .removed = p[1] }));
}
}
/// Card-escape `token` into `scratch` (grown as needed) and return the
/// escaped slice, valid until the next call.
fn escaped(gpa: std.mem.Allocator, scratch: *std.ArrayList(u8), token: []const u8) error{OutOfMemory}![]const u8 {
try scratch.resize(gpa, token.len * 2); // ยง4.3: an escaped byte doubles, nothing shrinks
return artifact.encodeEscapes(token, scratch.items);
}