Skip to content
zit Artifact [a599f8ed3e]
Login

Artifact [a599f8ed3e]

Artifact a599f8ed3eb14f6acc208d7770e641d1e04960c94f178cd440a7961b5096ef27:


//! `zit crosslink`: stream the derived per-check-in records.

const std = @import("std");
const zit = @import("zit");
const cli = @import("zit").cli;
const logstore = @import("zit").logstore;
const Repository = @import("zit").Repository;
const artifact = zit.artifact;
const diagnostics = @import("../diagnostics.zig");
const help_pages = @import("../help.zig");
const stateOom = diagnostics.stateOom;
const repoOpenFail = diagnostics.repoOpenFail;
const openRepo = diagnostics.openRepo;

/// Set from `$ZIT_LINES_WHOLE_TREE` at the same seam: `crosslink --lines`
/// reads each old side from the parent's whole file set rather than by
/// name, so the differential test can prove the two count the same.
pub var lines_whole_tree: bool = false;

// ---------------------------------------------------------------------------
// zit crosslink: stream the derived per-check-in records.
// ---------------------------------------------------------------------------

/// The stream-format version `runCrosslink` emits as its first line. Bump it
/// when a line's meaning changes; consumers refuse versions they do not know.
const crosslink_stream_version = "crosslink 1";

pub fn runCrosslink(
    gpa: std.mem.Allocator,
    io: std.Io,
    arena: std.mem.Allocator,
    verb_args: []const []const u8,
    streams: cli.Streams,
    amb: cli.Ambient,
) cli.Exit {
    var bypass = false;
    var lines = false;
    for (verb_args) |tok| {
        if (std.mem.eql(u8, tok, "-h") or std.mem.eql(u8, tok, "--help")) {
            streams.out.writeAll(help_pages.crosslink_help) catch {};
            return .ok;
        } else if (std.mem.eql(u8, tok, "--no-cache")) {
            bypass = true;
        } else if (std.mem.eql(u8, tok, "--lines")) {
            lines = true;
        } else if (tok.len > 1 and tok[0] == '-') {
            return cli.usageError(streams, "crosslink", .unknown_option, tok);
        } else {
            return cli.usageError(streams, "crosslink", .extra_argument, tok);
        }
    }

    // The open heals a stale cache (the extension path), so the
    // cache-served branch below almost always finds a body covering the whole
    // log. `--no-cache` opens with the bypass, so nothing below can be
    // answered by a sidecar โ€” the audit path this verb's byte-equality is
    // pinned against.
    var repo: Repository = undefined;
    var root_dir: std.Io.Dir = undefined;
    const rc = openRepo(gpa, io, arena, streams, "crosslink", amb.repo_override, bypass, !bypass, &repo, &root_dir);
    if (rc != .ok) return rc;
    defer root_dir.close(io);
    defer repo.deinit(gpa);
    const log = repo.logStore(gpa) catch |e| return repoOpenFail(streams, "crosslink", e);

    // Serve from the cache's record body when it provably covers the whole
    // log โ€” same key + shun-fingerprint discipline as the extension path โ€”
    // else derive the records fresh from the log. Both paths emit through one
    // encoder, and the e2e pins them byte-identical.
    var derived: std.ArrayList(u8) = .empty;
    defer derived.deinit(gpa);
    var raw: ?zit.repocache.RawCache = null;
    defer if (raw) |*rw| rw.deinit();

    var body: []const u8 = undefined;
    served_from_cache: {
        if (!bypass) {
            if (zit.repocache.loadForExtension(io, repo.root_dir)) |raw_const| {
                raw = raw_const;
                const key = log.logKey() catch break :served_from_cache;
                if (raw.?.shun_fp == log.shunFingerprint() and std.meta.eql(raw.?.key, key)) {
                    body = raw.?.body;
                    break :served_from_cache;
                }
                raw.?.deinit();
                raw = null;
            }
        }
        zit.repoview.captureRecords(gpa, log, 0, std.math.maxInt(u64), &derived) catch |e| switch (e) {
            error.OutOfMemory => return stateOom(streams, "crosslink"),
            else => {
                streams.err.writeAll("zit crosslink: the store could not be read\n") catch {};
                return .failure;
            },
        };
        body = derived.items;
    }

    if (!lines) return crosslinkEmit(gpa, streams, body, null);

    // Line counts come from their own sidecar, never from the record body:
    // counting reads every changed blob, and the body is built by any verb's
    // first open. Only `--lines` pays, and only once per check-in.
    // `--no-cache` counts everything afresh and stores nothing.
    const shun_fp = log.shunFingerprint();
    var counter: LineCounter = .{ .repo = &repo, .shun_fp = shun_fp, .log_entries = log.entryCount(), .io = io, .persist = !bypass };
    defer counter.deinit(gpa);
    if (!bypass) {
        if (zit.repocache.tryLoadLineStats(gpa, io, repo.root_dir)) |stored| {
            counter.stored = stored;
            counter.reader = zit.linestats.Reader.open(stored);
            if (counter.reader) |r| if (r.shunFp() != shun_fp) {
                counter.shun_moved = true;
                const keep = counter.markShunned(gpa, arena, log, body) catch return stateOom(streams, "crosslink");
                if (!keep) counter.reader = null;
            };
        }
    }
    const rc_emit = crosslinkEmit(gpa, streams, body, &counter);
    if (!bypass and rc_emit == .ok) {
        if (counter.writer) |w| {
            zit.repocache.storeLineStats(io, repo.root_dir, w.out.items);
        } else if ((counter.retried or counter.shun_moved) and counter.reader != null) {
            // Every entry was met and nothing needed rewriting: record the
            // shun table and the log the entries now stand for, so the next
            // run neither re-checks the shun nor retries the placeholders
            // until either moves again.
            if (counter.reader.?.restamped(gpa, shun_fp, counter.log_entries)) |bytes| {
                defer gpa.free(bytes);
                zit.repocache.storeLineStats(io, repo.root_dir, bytes);
            } else |_| {}
        }
    }
    return rc_emit;
}

/// Each changed file's added and removed lines against the primary parent,
/// counted the way `zit diff --numstat --checkin` counts them: the parent's
/// blob at the same path against the check-in's. Served from the stored
/// counts when an entry lines up with the record, counted otherwise.
///
/// While every record finds its stored entry next in line, nothing is
/// written. The first record that does not (a new check-in, a placeholder
/// that now counts, an entry out of order) starts `writer` with the stored
/// entries before it, and from then on every record is appended; the result
/// is stored when the stream ends.
const LineCounter = struct {
    repo: *Repository,
    shun_fp: u32,
    /// The log's entry count now, stamped on what is stored.
    log_entries: u64,
    io: std.Io,
    /// A placeholder was counted again because the log grew.
    retried: bool = false,
    /// The shun table moved since the counts were stored: an entry for a
    /// check-in that touches a path in `shunned_paths` is counted again.
    shun_moved: bool = false,
    /// Every path a shunned blob was ever written to. A shunned blob reaches
    /// a count as the new side of the check-in that wrote it, or as the old
    /// side of a later one that changed the same path; both touch the path.
    shunned_paths: std.StringHashMapUnmanaged(void) = .empty,
    /// Store what was counted (false under `--no-cache`).
    persist: bool,
    /// Check-ins counted since the writer was last stored. A first run over
    /// a long history stores its prefix every `flush_every`, so a run cut
    /// short (a timeout, a restart) is resumed rather than repeated.
    unflushed: usize = 0,
    stored: ?[]u8 = null,
    reader: ?zit.linestats.Reader = null,
    writer: ?zit.linestats.Writer = null,
    named: Repository.NamedMemo = .{},
    /// Every blob a count reads, whole, under the cache's byte bound -- not
    /// only the delta bases it keeps by default. A file's old side is
    /// usually the new side of an earlier check-in on its line, read moments
    /// before.
    plain: logstore.PlaintextCache = .{ .memoize_undeltaed = true },
    scratch: std.ArrayList(zit.linestats.Count) = .empty,

    const Count = zit.linestats.Count;
    const flush_every: usize = 4096;

    fn deinit(self: *LineCounter, gpa: std.mem.Allocator) void {
        if (self.reader) |*r| r.deinit(gpa);
        if (self.stored) |b| gpa.free(b);
        if (self.writer) |*w| w.deinit(gpa);
        self.named.deinit(gpa);
        self.shunned_paths.deinit(gpa);
        self.plain.deinit(gpa);
        self.scratch.deinit(gpa);
    }

    /// The counts for `rec`'s files, one per file in order. The slice is
    /// valid until the next call.
    fn forRecord(self: *LineCounter, gpa: std.mem.Allocator, rec: *const zit.repoview.DecodedRecord) error{OutOfMemory}![]const Count {
        // Where this record's entry starts if the stored entries are still
        // being met in order: everything before it is what the writer would
        // hold, should it have to start here.
        const in_order_at: ?usize = if (self.reader) |r| (if (r.inOrder()) r.at else null) else null;
        const found: zit.linestats.Reader.Found = if (self.reader) |*r| try r.find(gpa, rec.hash, &self.scratch) else .miss;
        // A placeholder against the log it was counted on stays one: the
        // content it lacked has not arrived.
        if (found == .recount and self.reader.?.logEntries() == self.log_entries) {
            self.scratch.clearRetainingCapacity();
            try self.scratch.appendNTimes(gpa, Count.unknown, rec.files.items.len);
            if (self.writer == null and !self.reader.?.inOrder()) try self.startWriter(gpa, in_order_at);
            if (self.writer) |*w| try w.append(gpa, rec.hash, self.scratch.items);
            return self.scratch.items;
        }
        if (found == .recount) self.retried = true;
        const usable = found == .hit and self.scratch.items.len == rec.files.items.len and !self.touchesShunned(rec);
        if (usable) {
            if (self.writer == null and !self.reader.?.inOrder()) try self.startWriter(gpa, in_order_at);
            if (self.writer) |*w| try w.append(gpa, rec.hash, self.scratch.items);
            return self.scratch.items;
        }
        self.scratch.clearRetainingCapacity();
        try self.scratch.appendNTimes(gpa, Count.unknown, rec.files.items.len);
        try self.count(gpa, rec, self.scratch.items);
        const complete = for (self.scratch.items) |c| {
            if (c.eql(Count.unknown)) break false;
        } else true;
        // A placeholder that still cannot be counted, met in order, changes
        // nothing that is stored.
        const unchanged = found == .recount and !complete and self.reader.?.inOrder();
        if (self.writer == null and !unchanged) try self.startWriter(gpa, in_order_at);
        if (self.writer) |*w| {
            try w.append(gpa, rec.hash, self.scratch.items);
            self.unflushed += 1;
            if (self.persist and self.unflushed >= flush_every) {
                zit.repocache.storeLineStats(self.io, self.repo.root_dir, w.out.items);
                self.unflushed = 0;
            }
        }
        return self.scratch.items;
    }

    /// Collect `shunned_paths` from `body`. False when the stored counts
    /// must all go: a check-in whose changed files are unknown may have
    /// written a shunned blob to a path nothing else records.
    fn markShunned(self: *LineCounter, gpa: std.mem.Allocator, arena: std.mem.Allocator, log: *logstore.LogStore, body: []const u8) error{OutOfMemory}!bool {
        if (log.shunList().len == 0) return true;
        var it = zit.repoview.RecordIter.init(body);
        while (it.next(gpa) catch return false) |rec_const| {
            var rec = rec_const;
            defer rec.deinit(gpa);
            if (rec.kind != .manifest) continue;
            if (!rec.files_known) return false;
            for (rec.files.items) |f| {
                const h = f.hash orelse continue;
                if (!isShunnedHex(log, h)) continue;
                if (!self.shunned_paths.contains(f.path)) try self.shunned_paths.put(gpa, try arena.dupe(u8, f.path), {});
            }
        }
        return true;
    }

    fn touchesShunned(self: *const LineCounter, rec: *const zit.repoview.DecodedRecord) bool {
        if (!self.shun_moved or self.shunned_paths.count() == 0) return false;
        for (rec.files.items) |f| {
            if (self.shunned_paths.contains(f.path)) return true;
        }
        return false;
    }

    fn isShunnedHex(log: *logstore.LogStore, hex: []const u8) bool {
        var buf: [logstore.max_hash_len]u8 = undefined;
        if (hex.len % 2 != 0 or hex.len / 2 > buf.len) return false;
        const digest = std.fmt.hexToBytes(buf[0 .. hex.len / 2], hex) catch return false;
        return logstore.isCompleteDigest(digest.len) and log.isShunned(digest);
    }

    /// Start the writer with the stored entries before `prefix_end` (none
    /// without a stored blob or once the order was lost; then every entry
    /// before this record was appended as it was met, so there is none).
    fn startWriter(self: *LineCounter, gpa: std.mem.Allocator, prefix_end: ?usize) error{OutOfMemory}!void {
        var w = try zit.linestats.Writer.init(gpa, self.shun_fp, self.log_entries);
        errdefer w.deinit(gpa);
        if (prefix_end) |end| try w.out.appendSlice(gpa, self.stored.?[zit.linestats.header_len..end]);
        self.writer = w;
    }

    /// Fill `out` (pre-set to unknown) for every file both of whose sides
    /// can be read.
    fn count(self: *LineCounter, gpa: std.mem.Allocator, rec: *const zit.repoview.DecodedRecord, out: []Count) error{OutOfMemory}!void {
        const log = self.repo.logStore(gpa) catch |e| switch (e) {
            error.OutOfMemory => return error.OutOfMemory,
            else => return,
        };
        // The parent-side blob of each path, null where the parent lacks it.
        var arena_state = std.heap.ArenaAllocator.init(gpa);
        defer arena_state.deinit();
        const arena = arena_state.allocator();
        const olds = try arena.alloc(Repository.NamedHash, rec.files.items.len);
        for (rec.files.items, olds) |f, *o| o.* = .{ .name = f.path };
        if (rec.parents.items.len != 0) {
            const prid = self.repo.lookup(rec.parents.items[0]) orelse return;
            const found = (if (lines_whole_tree) self.repo.hashesNamedWhole(gpa, arena, prid, olds, &self.named) else self.repo.hashesNamed(gpa, arena, prid, olds, &self.named, &self.plain)) catch |e| switch (e) {
                error.OutOfMemory => return error.OutOfMemory,
                else => return,
            };
            if (!found) return;
        }
        for (rec.files.items, olds, out) |f, o, *slot| {
            const old_hash = o.hash;
            if (old_hash != null and f.hash != null and std.mem.eql(u8, old_hash.?, f.hash.?)) {
                slot.* = Count.text(0, 0); // a mode-only change: same blob both sides
                continue;
            }
            const old = (try self.side(gpa, log, old_hash)) orelse continue;
            defer gpa.free(old);
            const new = (try self.side(gpa, log, if (f.disp == .deleted) null else f.hash)) orelse continue;
            defer gpa.free(new);
            if (zit.textdiff.isBinary(old) or zit.textdiff.isBinary(new)) {
                slot.* = Count.binary;
            } else {
                const lc = try zit.textdiff.lineCounts(gpa, old, new, .{});
                slot.* = Count.text(lc.added, lc.removed);
            }
        }
    }

    /// One side's bytes: empty for an absent side, null when the store
    /// cannot produce the blob. Read straight from the log the way the derive
    /// reads manifests: each frame is CRC-checked, and re-hashing every blob
    /// of a history to name it again was most of a first run. Caller frees.
    fn side(self: *LineCounter, gpa: std.mem.Allocator, log: *logstore.LogStore, hash: ?[]const u8) error{OutOfMemory}!?[]u8 {
        const h = hash orelse return try gpa.alloc(u8, 0);
        var digest_buf: [logstore.max_hash_len]u8 = undefined;
        if (h.len % 2 != 0 or h.len / 2 > digest_buf.len) return null;
        const digest = std.fmt.hexToBytes(digest_buf[0 .. h.len / 2], h) catch return null;
        const rid = log.lookup(digest) orelse return null;
        return log.contentCached(gpa, rid, &self.plain) catch |e| switch (e) {
            error.OutOfMemory => return error.OutOfMemory,
            else => return null,
        };
    }
};

/// Emit one record body as the versioned text stream. Free-text tokens (user,
/// tag names and values, paths) are card-escaped exactly as an artifact would
/// carry them, so every emitted line is newline-safe and a consumer reuses
/// the artifact-format decoder it already has. The comment is length-framed
/// raw bytes, the one field escaping cannot serve (it is free prose).
fn crosslinkEmit(gpa: std.mem.Allocator, streams: cli.Streams, body: []const u8, counter: ?*LineCounter) cli.Exit {
    var scratch: std.ArrayList(u8) = .empty;
    defer scratch.deinit(gpa);
    var lines_buf: [lines_line_max]u8 = undefined;

    streams.out.print("{s}\n", .{crosslink_stream_version}) catch return .failure;
    var it = zit.repoview.RecordIter.init(body);
    while (it.next(gpa) catch {
        streams.err.writeAll("zit crosslink: the record body is corrupt; re-run with --no-cache\n") catch {};
        return .failure;
    }) |rec_const| {
        var rec = rec_const;
        defer rec.deinit(gpa);
        if (rec.kind != .manifest) continue; // control artifacts: not check-ins

        streams.out.print("checkin {s} {d}", .{ rec.hash, rec.mtime }) catch return .failure;
        for (rec.parents.items) |parent| streams.out.print(" {s}", .{parent}) catch return .failure;
        streams.out.writeAll("\n") catch return .failure;
        if (rec.user.len != 0) {
            streams.out.print("user {s}\n", .{escaped(gpa, &scratch, rec.user) catch return .failure}) catch return .failure;
        }
        streams.out.print("comment {d}\n", .{rec.comment.len}) catch return .failure;
        streams.out.writeAll(rec.comment) catch return .failure;
        streams.out.writeAll("\n") catch return .failure;
        for (rec.tags.items) |t| {
            const sigil: u8 = switch (t.type) {
                .singleton => '+',
                .cancel => '-',
                .propagate => '*',
            };
            streams.out.print("tag {c}{s}", .{ sigil, escaped(gpa, &scratch, t.name) catch return .failure }) catch return .failure;
            if (t.target) |target| {
                streams.out.print(" {s}", .{target}) catch return .failure;
            } else {
                streams.out.writeAll(" *") catch return .failure;
            }
            if (t.value) |v| {
                streams.out.print(" {s}", .{escaped(gpa, &scratch, v) catch return .failure}) catch return .failure;
            }
            streams.out.writeAll("\n") catch return .failure;
        }
        for (rec.picks.items) |pk| {
            const sign: u8 = switch (pk.sign) {
                .cherrypick => '+',
                .backout => '-',
            };
            streams.out.print("cherrypick {c}{s}\n", .{ sign, pk.target }) catch return .failure;
        }
        if (rec.files_known) {
            // Without `--lines` nothing is counted and no `lines` line is printed.
            const counts: ?[]const LineCounter.Count = if (counter) |lc| (lc.forRecord(gpa, &rec) catch return stateOom(streams, "crosslink")) else null;
            for (rec.files.items, 0..) |f, i| {
                streams.out.print("file {c} {s}\n", .{ @backingInt(f.disp), escaped(gpa, &scratch, f.path) catch return .failure }) catch return .failure;
                if (counts) |cs| {
                    const c = cs[i];
                    if (c.eql(LineCounter.Count.binary)) {
                        streams.out.writeAll("lines - -\n") catch return .failure;
                    } else if (!c.eql(LineCounter.Count.unknown)) {
                        streams.out.writeAll(linesLine(&lines_buf, c)) catch return .failure;
                    }
                }
                // The rename the check-in recorded, after the file it names,
                // so a consumer that knows only A/D lines reads what it
                // always read and one that knows this line carries the
                // file's identity across it.
                if (f.prior) |old| {
                    streams.out.print("rename {s}", .{escaped(gpa, &scratch, old) catch return .failure}) catch return .failure;
                    streams.out.print(" {s}\n", .{escaped(gpa, &scratch, f.path) catch return .failure}) catch return .failure;
                }
            }
        } else {
            streams.out.writeAll("files unknown\n") catch return .failure;
        }
        streams.out.writeAll("end\n") catch return .failure;
    }
    return .ok;
}

const lines_line_max = "lines ".len + 10 + 1 + 10 + 1;

/// `lines ADDED REMOVED\n` for a text count, formatted by hand: a long
/// history prints one per changed file, and the general formatter was most
/// of what printing them cost.
fn linesLine(buf: *[lines_line_max]u8, c: zit.linestats.Count) []const u8 {
    var n: usize = 0;
    @memcpy(buf[0.."lines ".len], "lines ");
    n += "lines ".len;
    n += decimal(buf[n..], c.added);
    buf[n] = ' ';
    n += 1;
    n += decimal(buf[n..], c.removed);
    buf[n] = '\n';
    return buf[0 .. n + 1];
}

/// Write `v` in decimal at the start of `out`; returns the digit count.
fn decimal(out: []u8, v: u32) usize {
    var tmp: [10]u8 = undefined;
    var i: usize = tmp.len;
    var x = v;
    while (true) {
        i -= 1;
        tmp[i] = '0' + @as(u8, @intCast(x % 10));
        x /= 10;
        if (x == 0) break;
    }
    const len = tmp.len - i;
    @memcpy(out[0..len], tmp[i..]);
    return len;
}

test "linesLine formats like the general formatter" {
    var buf: [lines_line_max]u8 = undefined;
    for ([_][2]u32{ .{ 0, 0 }, .{ 12, 3 }, .{ 9, 10 }, .{ std.math.maxInt(u32) - 2, 1 } }) |p| {
        var want: [lines_line_max]u8 = undefined;
        const w = try std.fmt.bufPrint(&want, "lines {d} {d}\n", .{ p[0], p[1] });
        try std.testing.expectEqualStrings(w, linesLine(&buf, .{ .added = p[0], .removed = p[1] }));
    }
}

/// Card-escape `token` into `scratch` (grown as needed) and return the
/// escaped slice, valid until the next call.
fn escaped(gpa: std.mem.Allocator, scratch: *std.ArrayList(u8), token: []const u8) error{OutOfMemory}![]const u8 {
    try scratch.resize(gpa, token.len * 2); // ยง4.3: an escaped byte doubles, nothing shrinks
    return artifact.encodeEscapes(token, scratch.items);
}