Artifact [c78171a790]

Artifact c78171a79048e3c8a4cb7d67b3f1bbf8abe73016840a3af399aab2b989968de2:

• File src/archive.zig — part of check-in [580d0691c7] at 2026-08-17 14:41:42 on branch trunk — tarball: write a check-in's files as a reproducible gzip tar (#95)

  The effective-set walk already resolves sub-manifests and delta chains in canonical order, so this is that walk with an archive writer. Members carry the F card's permission, so an executable stays executable. Two runs of one version are byte-identical: history supplies the order and the timestamps, and no uid, gid, or build host is recorded. A file the store does not hold is refused by name -- a partial archive looks exactly like success. (user: dmestas, size: 19904) [annotate] [blame] [check-ins using]

//! Archive export (#95): a check-in's effective file set written to a tar or
//! zip stream instead of to the filesystem.
//!
//! This module is the FORMAT and the POLICY, deliberately without I/O beyond
//! the caller's writer: header encoding, member selection, and the top-level
//! directory name are pure functions of their inputs, so the properties that
//! matter can be tested without a repository. Enumerating the check-in and
//! reading blobs stays in the command, where those handles live.
//!
//! Every archive this module writes is REPRODUCIBLE: identical inputs give
//! identical bytes, on any machine, in any order of runs. That is not a
//! side effect to be preserved by care, it is the property that makes an
//! archive usable in a release pipeline, and it is easy to lose by accident.
//! Three things secure it, and each is load-bearing:
//!
//! 1. **Member order is the check-in's own.** The effective-set walk yields F
//!    cards in canonical filename order (artifact §4.5.3), so the caller never
//!    sorts and never depends on a directory read.
//! 2. **Every timestamp comes from history**, never from the clock: members
//!    carry the check-in's own D-card time, which `SOURCE_DATE_EPOCH` already
//!    pins for reproducible builds.
//! 3. **No identity is embedded.** uid/gid 0, and the gzip container's header
//!    is a fixed constant with a zeroed mtime and no host field, so neither
//!    the operator nor the build machine leaks into the bytes.

const std = @import("std");
const artifact = @import("artifact.zig");
const control_paths = @import("control_paths.zig");
const ignore = @import("ignore.zig");
const managed = @import("managed.zig");

const testing = std.testing;
const Writer = std.Io.Writer;

/// tar's fixed block. Every header is one block and every member's content is
/// padded up to a multiple of it.
pub const tar_block: usize = 512;

/// ustar's own name fields: `name` is 100 bytes, `prefix` 155, joined by an
/// implied '/'. A path that fits is written as plain ustar and read by every
/// tar in existence; a longer one needs the GNU long-name escape below, since
/// `managed.max_path_len` (4096) is far past what ustar can spell.
pub const ustar_name_len: usize = 100;
pub const ustar_prefix_len: usize = 155;

/// The uid/gid every member carries. Zero, because an archive that recorded
/// the exporting operator's numeric id would differ between two machines
/// producing otherwise identical output — the reproducibility property this
/// module exists to hold.
pub const archive_uid: u32 = 0;
pub const archive_gid: u32 = 0;

comptime {
    // The GNU long-name escape is what covers the gap between ustar's spelling
    // and a real tracked path; if the bound ever fit, the escape would be dead
    // code and this module should lose it rather than keep it untested.
    std.debug.assert(managed.max_path_len > ustar_name_len + 1 + ustar_prefix_len);
}

pub const Error = error{
    /// A member's path names a reserved control root. `add` and every import
    /// path already refuse to create such an F card, so seeing one here means
    /// the manifest is not one this tool could have written — fail closed
    /// rather than silently drop it, which is how a partial archive that
    /// looks complete gets made.
    ReservedPath,
    /// A path longer than `managed.max_path_len`, which no legal F card can
    /// carry and no escape here is sized for.
    PathTooLong,
};

/// One member of the archive, in the order the effective-set walk yielded it.
pub const Entry = struct {
    /// Repository-relative path, WITHOUT the top-level directory — the caller
    /// supplies that once and this module joins it, so the two cannot drift.
    path: []const u8,
    mode: artifact.FileMode,
};

/// tar's typeflag byte for a mode. Symlinks stay symlinks (`2`) rather than
/// being flattened into a regular file holding their target text, so a tree
/// extracted from an archive matches one `zit open` would have written.
fn typeFlag(mode: artifact.FileMode) u8 {
    return switch (mode) {
        .regular, .executable => '0',
        .symlink => '2',
    };
}

/// The permission bits a mode earns. Deliberately a fixed table rather than
/// anything read from the filesystem: the F card's mode is the whole of what
/// history records, so it is the whole of what the archive can honestly claim.
fn permBits(mode: artifact.FileMode) u32 {
    return switch (mode) {
        .regular => 0o644,
        .executable => 0o755,
        .symlink => 0o777,
    };
}

/// Whether a member survives the `--include` / `--exclude` filters.
///
/// **Exclude wins**, matching what fossil documents for the same two options.
/// The rule is copied rather than decided afresh precisely because it is the
/// kind of detail that silently differs between two tools and then surprises
/// someone holding an archive that is missing a file they asked for.
///
/// An absent `include` means "everything not excluded"; an empty-but-present
/// one means the operator asked for a glob list and gave none, which selects
/// nothing. Both globs read through the shared fossil-dialect list matcher, so
/// `--exclude` here and `ignore-glob` in a setting cannot drift into separate
/// dialects.
pub fn selected(path: []const u8, include: ?[]const u8, exclude: ?[]const u8) bool {
    if (exclude) |globs| {
        if (ignore.listMatches(globs, path)) return false;
    }
    if (include) |globs| return ignore.listMatches(globs, path);
    return true;
}

/// Validate one effective-set path for archiving. Separate from `selected` so
/// a refusal is never confused with a filter miss: one is the operator's own
/// glob doing its job, the other is a repository that should not exist.
pub fn admit(path: []const u8) Error!void {
    if (path.len > managed.max_path_len) return error.PathTooLong;
    if (control_paths.isReservedRoot(path)) return error.ReservedPath;
}

/// Longest top-level directory name this module will join onto a member path
/// and still guarantee the result is spellable. Bounded so `--name` cannot
/// push a legal path past what the long-name escape is sized for.
pub const max_top_dir_len: usize = 255;

/// Archive suffixes stripped when deriving a default top-level directory from
/// an output filename. Ordered longest-first so `.tar.gz` wins over `.gz`.
const archive_suffixes = [_][]const u8{ ".tar.gz", ".tar.bz2", ".tgz", ".tar", ".zip" };

/// The default top-level directory, when `--name` was not given.
///
/// Fossil derives this from the project NAME, which zit has no equivalent of —
/// `project-code` is a sync identity, not a human label, and an archive whose
/// top directory was a 40-hex sync code would be worse than useless. So the
/// derivation is: the output filename's basename with archive suffixes
/// stripped (`release-1.2.tar.gz` gives `release-1.2`, which is what an
/// operator naming an output file almost always meant), falling back to the
/// check-in's own short artifact id when there is no usable output name —
/// `--list`, `/dev/null`, or the empty string.
///
/// `out` must hold `max_top_dir_len` bytes; the result borrows it.
pub fn defaultTopDir(output_path: []const u8, checkin_prefix: []const u8, out: []u8) []const u8 {
    std.debug.assert(out.len >= max_top_dir_len);
    std.debug.assert(checkin_prefix.len >= 1);

    const base = std.fs.path.basename(output_path);
    var name = base;
    for (archive_suffixes) |suffix| {
        // `>=`, not `>`: a name that IS a suffix (`.tar.gz`) strips to empty
        // and falls back below, rather than standing as a top directory
        // literally called `.tar.gz`.
        if (name.len >= suffix.len and std.ascii.endsWithIgnoreCase(name, suffix)) {
            name = name[0 .. name.len - suffix.len];
            break;
        }
    }
    // `/dev/null` basenames to "null", which would be a silently wrong top
    // directory rather than an obviously missing one; treat every unusable
    // output name the same way and fall back to the check-in.
    if (name.len == 0 or name.len > max_top_dir_len or std.mem.eql(u8, base, "null")) {
        const n = @min(checkin_prefix.len, max_top_dir_len);
        @memcpy(out[0..n], checkin_prefix[0..n]);
        return out[0..n];
    }
    @memcpy(out[0..name.len], name);
    return out[0..name.len];
}

/// Split `path` into ustar's `prefix` and `name` fields, or null when it
/// cannot be spelled that way. ustar joins them with an implied '/', so the
/// split must land ON a separator and neither half may overflow its field.
/// Returns the longest legal prefix, which is what maximises the number of
/// paths that avoid the long-name escape.
fn ustarSplit(path: []const u8) ?struct { prefix: []const u8, name: []const u8 } {
    if (path.len <= ustar_name_len) return .{ .prefix = "", .name = path };
    if (path.len > ustar_prefix_len + 1 + ustar_name_len) return null;
    // Walk separators from the latest legal split backwards: the name half
    // must fit in 100 bytes, the prefix half in 155.
    var cut: ?usize = null;
    var i: usize = 0;
    while (i < path.len) : (i += 1) {
        if (path[i] != '/') continue;
        const prefix_len = i;
        const name_len = path.len - i - 1;
        if (prefix_len == 0 or prefix_len > ustar_prefix_len) continue;
        if (name_len == 0 or name_len > ustar_name_len) continue;
        cut = i; // keep scanning; a later cut shortens the name half
    }
    const c = cut orelse return null;
    return .{ .prefix = path[0..c], .name = path[c + 1 ..] };
}

/// Write `value` as tar's zero-padded octal with a trailing NUL, the encoding
/// every field but the checksum uses.
fn octal(field: []u8, value: u64) void {
    std.debug.assert(field.len >= 2);
    const digits = field.len - 1;
    var v = value;
    var i: usize = digits;
    while (i > 0) {
        i -= 1;
        field[i] = '0' + @as(u8, @intCast(v & 7));
        v >>= 3;
    }
    std.debug.assert(v == 0); // caller's value must fit the field
    field[digits] = 0;
}

/// The ustar header checksum: the sum of every header byte with the checksum
/// field itself read as eight spaces. Written as six octal digits, a NUL, and
/// a space — the historical encoding every reader accepts.
fn checksum(header: *[tar_block]u8) void {
    @memset(header[148..156], ' ');
    var sum: u64 = 0;
    for (header) |b| sum += b;
    octal(header[148..155], sum);
    header[155] = ' ';
}

/// Encode one ustar header block. `link_target` is the symlink destination
/// and must be null for every other mode.
fn ustarHeader(
    name: []const u8,
    prefix: []const u8,
    size: u64,
    mode: artifact.FileMode,
    mtime: u64,
    link_target: ?[]const u8,
    type_flag: u8,
    out: *[tar_block]u8,
) void {
    std.debug.assert(name.len <= ustar_name_len);
    std.debug.assert(prefix.len <= ustar_prefix_len);
    @memset(out, 0);
    @memcpy(out[0..name.len], name);
    octal(out[100..108], permBits(mode));
    octal(out[108..116], archive_uid);
    octal(out[116..124], archive_gid);
    octal(out[124..136], size);
    octal(out[136..148], mtime);
    out[156] = type_flag;
    if (link_target) |target| {
        std.debug.assert(target.len <= ustar_name_len);
        @memcpy(out[157..][0..target.len], target);
    }
    @memcpy(out[257..263], "ustar\x00");
    @memcpy(out[263..265], "00");
    // uname/gname stay EMPTY rather than "root": a name is a claim about the
    // extracting system's account table that this archive has no basis for,
    // and readers fall back to the numeric id, which is the honest answer.
    @memcpy(out[345..][0..prefix.len], prefix);
    checksum(out);
}

/// Pad the current member up to tar's block boundary.
fn padTo(w: *Writer, written: u64) Writer.Error!void {
    const rem = written % tar_block;
    if (rem == 0) return;
    var zeros: [tar_block]u8 = @splat(0);
    try w.writeAll(zeros[0 .. tar_block - rem]);
}

/// GNU's long-name escape: a pseudo-member of type 'L' whose CONTENT is the
/// real path, immediately followed by the real header carrying a truncated
/// name. Chosen over pax extended headers because tracked paths reach 4096
/// bytes and this is the escape the widest set of readers implements —
/// GNU tar, bsdtar, and Python's tarfile all honour it.
const gnu_longname_flag: u8 = 'L';
const gnu_longname_name = "././@LongLink";

/// Write one archive member's header, then `size` bytes of content the caller
/// streams, then the block padding. `path` already includes the top-level
/// directory. The caller writes content between this call and `endTarMember`.
pub fn beginTarMember(
    w: *Writer,
    path: []const u8,
    size: u64,
    mode: artifact.FileMode,
    mtime: u64,
    link_target: ?[]const u8,
) Writer.Error!void {
    var header: [tar_block]u8 = undefined;
    if (ustarSplit(path)) |split| {
        ustarHeader(split.name, split.prefix, size, mode, mtime, link_target, typeFlag(mode), &header);
        try w.writeAll(&header);
        return;
    }
    // Too long for ustar's two fields: emit the name as a 'L' member's
    // content, then a header whose own name field is a truncated stand-in.
    // A reader that does not know 'L' extracts a file literally called
    // `././@LongLink`, which is visibly wrong rather than silently truncated.
    ustarHeader(gnu_longname_name, "", path.len + 1, .regular, 0, null, gnu_longname_flag, &header);
    try w.writeAll(&header);
    try w.writeAll(path);
    try w.writeByte(0);
    try padTo(w, path.len + 1);
    ustarHeader(path[0..ustar_name_len], "", size, mode, mtime, link_target, typeFlag(mode), &header);
    try w.writeAll(&header);
}

/// Close a member started by `beginTarMember` after `size` content bytes.
pub fn endTarMember(w: *Writer, size: u64) Writer.Error!void {
    try padTo(w, size);
}

/// tar's end-of-archive marker: two zero blocks.
pub fn writeTarEnd(w: *Writer) Writer.Error!void {
    var zeros: [tar_block * 2]u8 = @splat(0);
    try w.writeAll(&zeros);
}

test "archive selected: exclude wins over include, absent include admits" {
    // The precedence rule copied from fossil, pinned because it is the detail
    // that silently differs between tools.
    try testing.expect(selected("src/a.zig", null, null));
    try testing.expect(selected("src/a.zig", "src/*", null));
    try testing.expect(!selected("src/a.zig", null, "src/*"));
    // Matching BOTH is excluded — the whole point of the rule.
    try testing.expect(!selected("src/a.zig", "src/*", "src/*"));
    try testing.expect(!selected("src/a.zig", "*.zig", "src/a.zig"));
    // An include that matches nothing selects nothing.
    try testing.expect(!selected("src/a.zig", "docs/*", null));
    // A present-but-empty include is a list with no patterns: it matches
    // nothing, which is different from having asked for no filter at all.
    try testing.expect(!selected("src/a.zig", "", null));
    try testing.expect(selected("src/a.zig", null, ""));
}

test "archive admit: a reserved control root is refused, not filtered" {
    try admit("src/a.zig");
    try admit("README");
    try testing.expectError(error.ReservedPath, admit(".zit"));
    try testing.expectError(error.ReservedPath, admit(".zit/store"));
    try testing.expectError(error.ReservedPath, admit(".zit-checkout"));
    try testing.expectError(error.ReservedPath, admit(".zit-cache/x"));
    // A name that merely starts with the same bytes is a normal file.
    try admit(".zitten");
}

test "archive defaultTopDir: derived from the output name, else the check-in" {
    var buf: [max_top_dir_len]u8 = undefined;
    try testing.expectEqualStrings("release-1.2", defaultTopDir("release-1.2.tar.gz", "abc123", &buf));
    try testing.expectEqualStrings("release-1.2", defaultTopDir("out/release-1.2.zip", "abc123", &buf));
    try testing.expectEqualStrings("release-1.2", defaultTopDir("release-1.2.tgz", "abc123", &buf));
    try testing.expectEqualStrings("snap", defaultTopDir("snap.tar", "abc123", &buf));
    // No recognised suffix: the basename stands as given.
    try testing.expectEqualStrings("snapshot", defaultTopDir("snapshot", "abc123", &buf));
    // The three unusable output names all fall back to the check-in, and
    // /dev/null must NOT become a top directory called "null".
    try testing.expectEqualStrings("abc123", defaultTopDir("", "abc123", &buf));
    try testing.expectEqualStrings("abc123", defaultTopDir("/dev/null", "abc123", &buf));
    // A suffix with nothing before it is not a name.
    try testing.expectEqualStrings("abc123", defaultTopDir(".tar.gz", "abc123", &buf));
}

test "archive ustarSplit: fits, splits on a separator, or refuses" {
    // Short enough for the name field alone.
    const short = ustarSplit("src/a.zig").?;
    try testing.expectEqualStrings("", short.prefix);
    try testing.expectEqualStrings("src/a.zig", short.name);

    // 100 bytes exactly still fits the name field.
    const exact = "a" ** 100;
    try testing.expectEqualStrings(exact, ustarSplit(exact).?.name);

    // Longer than 100 with a separator positioned so both halves fit: the
    // split lands on the separator and the halves rejoin to the original.
    const long = "d" ** 90 ++ "/" ++ "f" ** 40;
    const split = ustarSplit(long).?;
    try testing.expectEqualStrings("d" ** 90, split.prefix);
    try testing.expectEqualStrings("f" ** 40, split.name);

    // Over 100 with NO separator cannot be spelled by ustar at all.
    try testing.expect(ustarSplit("a" ** 120) == null);
    // A tail longer than the name field cannot be spelled either, however
    // early the separator sits.
    try testing.expect(ustarSplit("d/" ++ "f" ** 120) == null);
    // Past the joined bound entirely.
    try testing.expect(ustarSplit("d" ** 200 ++ "/" ++ "f" ** 90) == null);
}

test "archive octal and checksum: the encodings every tar reader expects" {
    var field: [8]u8 = undefined;
    octal(&field, 0o644);
    try testing.expectEqualStrings("0000644\x00", &field);
    octal(&field, 0);
    try testing.expectEqualStrings("0000000\x00", &field);

    // A header's checksum covers every byte with the field read as spaces, so
    // recomputing over the finished block must reproduce the stored value.
    var header: [tar_block]u8 = undefined;
    ustarHeader("a.txt", "", 3, .regular, 1_600_000_000, null, '0', &header);
    const stored = header[148..156].*;
    var sum: u64 = 0;
    for (header, 0..) |b, i| sum += if (i >= 148 and i < 156) ' ' else b;
    var expect: [8]u8 = undefined;
    octal(expect[0..7], sum);
    expect[7] = ' ';
    try testing.expectEqualSlices(u8, &expect, &stored);
    try testing.expectEqualStrings("ustar\x00", header[257..263]);
    try testing.expectEqualStrings("00", header[263..265]);
    // No identity is embedded: uid, gid, and both name fields stay zero.
    try testing.expectEqualStrings("0000000\x00", header[108..116]);
    try testing.expectEqualStrings("0000000\x00", header[116..124]);
    for (header[265..345]) |b| try testing.expectEqual(@as(u8, 0), b);
}

test "archive header modes: perms and typeflags follow the F card, not the disk" {
    var header: [tar_block]u8 = undefined;
    ustarHeader("x", "", 0, .regular, 0, null, typeFlag(.regular), &header);
    try testing.expectEqualStrings("0000644\x00", header[100..108]);
    try testing.expectEqual(@as(u8, '0'), header[156]);

    ustarHeader("x", "", 0, .executable, 0, null, typeFlag(.executable), &header);
    try testing.expectEqualStrings("0000755\x00", header[100..108]);
    try testing.expectEqual(@as(u8, '0'), header[156]);

    // A symlink stays a symlink and carries its target in the link field.
    ustarHeader("x", "", 0, .symlink, 0, "../target", typeFlag(.symlink), &header);
    try testing.expectEqual(@as(u8, '2'), header[156]);
    try testing.expectEqualStrings("../target", header[157..166]);
}