//! `zit tarball` / `zit zip` — export a check-in's effective file set as
//! an archive instead of a working tree.
//!
//! No new history logic lives here. `Repository.files` already walks a
//! check-in's effective set through sub-manifests and delta baselines, in
//! canonical filename order; `Repository.blob` already reconstructs a blob
//! through its delta chain and verifies it against its F-card id. This verb is
//! that same walk with a writer that appends to an archive, which is why it is
//! small: the format and the selection policy live in `archive.zig`, and what
//! is left here is the repository handle, the output file, and the diagnostics.
//!
//! The one property worth stating at this layer: **a member whose content the
//! store does not hold is a refusal, never an omission.** A sparse store
//! deliberately elides out-of-cone blobs, and an archive quietly missing those
//! files would be the failure mode that looks exactly like success — right
//! member count, plausible size, missing source. So the whole member set is
//! checked for presence BEFORE a single byte is written, and the refusal names
//! the files rather than reporting a count.
const std = @import("std");
const zit = @import("zit");
const cli = @import("zit").cli;
const archive = @import("zit").archive;
const artifact = @import("zit").artifact;
const Repository = @import("zit").Repository;
const main = @import("../main.zig");
const openRepo = main.openRepo;
const not_a_repo_msg = main.not_a_repo_msg;
/// Which archive format a run writes. The enumeration and selection are shared
/// whole; only the writer differs.
pub const Kind = enum {
tarball,
zip,
fn verb(self: Kind) []const u8 {
return switch (self) {
.tarball => "tarball",
.zip => "zip",
};
}
fn help(self: Kind) []const u8 {
return switch (self) {
.tarball => main.tarball_help,
.zip => main.zip_help,
};
}
};
/// How many members one archive may carry. The same bound the managed set
/// already imposes on a checkout, restated here so the collection loop below
/// is finite by inspection rather than by trusting the manifest it walked.
const max_members: usize = zit.managed.max_entries;
/// Largest single member. A blob is reconstructed whole into memory to be
/// written, exactly as `zit cat` does, so this is `cat`'s bound and not a new
/// policy — an archive is not a streaming decompressor.
const max_member_bytes: u64 = 1 << 32;
/// A selected member: its repository path, its mode, and its blob id. Paths
/// and hashes borrow the `FileList`, which outlives this slice.
const Member = struct {
path: []const u8,
mode: artifact.FileMode,
hash: []const u8,
};
pub fn runTarball(
gpa: std.mem.Allocator,
io: std.Io,
arena: std.mem.Allocator,
verb_args: []const []const u8,
streams: cli.Streams,
amb: cli.Ambient,
) cli.Exit {
return run(.tarball, gpa, io, arena, verb_args, streams, amb);
}
fn run(
kind: Kind,
gpa: std.mem.Allocator,
io: std.Io,
arena: std.mem.Allocator,
verb_args: []const []const u8,
streams: cli.Streams,
amb: cli.Ambient,
) cli.Exit {
const verb = kind.verb();
const parsed = switch (cli.parseArchiveArgs(verb_args)) {
.ok => |p| p,
.unknown_option => |t| return cli.usageError(streams, verb, .unknown_option, t),
.extra_argument => |t| return cli.usageError(streams, verb, .extra_argument, t),
.empty_value => |t| return cli.usageError(streams, verb, .empty_value, t),
.repeated_option => |t| return cli.usageError(streams, verb, .repeated_option, t),
};
if (parsed.help) {
streams.out.writeAll(kind.help()) catch {};
return .ok;
}
const version = parsed.version orelse
return cli.usageError(streams, verb, .no_checkin, "");
// OUTPUTFILE is required unless there is nothing to write.
if (parsed.output == null and !parsed.list)
return cli.usageError(streams, verb, .no_file, "");
if (parsed.name) |n| {
if (n.len > archive.max_top_dir_len or std.mem.indexOfScalar(u8, n, '/') != null) {
streams.err.print(
"zit {s}: --name must be a single directory name of at most {d} bytes\n",
.{ verb, archive.max_top_dir_len },
) catch {};
return .usage;
}
}
var repo: Repository = undefined;
var root_dir: std.Io.Dir = undefined;
const rc = openRepo(gpa, io, arena, streams, verb, amb.repo_override, parsed.no_cache, false, &repo, &root_dir);
if (rc != .ok) return rc;
defer root_dir.close(io);
defer repo.deinit(gpa);
const rid = repo.resolve(version, .checkin) catch |e| switch (e) {
error.NotFound => return fail(streams, verb, "check-in '{s}' not found", .{version}),
error.Ambiguous => return fail(streams, verb, "name '{s}' is ambiguous", .{version}),
error.OutOfMemory => return oom(streams, verb),
};
var list = (repo.files(gpa, rid) catch |e| return filesFail(streams, verb, e)) orelse
return fail(streams, verb, "check-in content is missing", .{});
defer list.deinit(gpa);
// Collect the selected members first, so nothing is written before the
// whole set is known to be admissible AND present.
var members: std.ArrayListUnmanaged(Member) = .empty;
defer members.deinit(gpa);
var it = list.iterator();
while (it.next()) |f| {
std.debug.assert(f.hash != null); // the walk suppresses delete markers
archive.admit(f.filename) catch |e| return admitFail(streams, verb, f.filename, e);
if (!archive.selected(f.filename, parsed.include, parsed.exclude)) continue;
if (members.items.len >= max_members)
return fail(streams, verb, "check-in carries more than {d} files", .{max_members});
members.append(gpa, .{
.path = f.filename,
.mode = artifact.fileModeOf(f.perm),
.hash = f.hash.?,
}) catch return oom(streams, verb);
}
// Presence, before any output. A sparse store interns an elided blob's
// name while holding no bytes, so this is a real question even when the
// manifest looks complete — and answering it costs a lookup per member,
// not an inflate.
var missing: usize = 0;
for (members.items) |m| {
const held = repo.blobPresent(gpa, m.hash) catch |e| return openFail(streams, verb, e);
if (held) continue;
if (missing == 0) {
streams.err.print(
"zit {s}: this store does not hold every file in the check-in; refusing to write a partial archive\n",
.{verb},
) catch {};
}
missing += 1;
// Name them rather than counting them: "3 files missing" sends the
// operator to guess which, and the remedy differs per path.
if (missing <= 20) streams.err.print("zit {s}: {s}\n", .{ verb, m.path }) catch {};
}
if (missing > 20) {
streams.err.print("zit {s}: ... and {d} more\n", .{ verb, missing - 20 }) catch {};
}
if (missing != 0) {
streams.err.print(
"zit {s}: widen the cone with 'zit cone add PATH' (which fetches), or archive from a dense store\n",
.{verb},
) catch {};
return .failure;
}
// The top-level directory: the operator's, or derived.
var name_buf: [archive.max_top_dir_len]u8 = undefined;
const checkin_hash = repo.hashOf(rid);
const top = parsed.name orelse archive.defaultTopDir(
parsed.output orelse "",
checkin_hash[0..@min(checkin_hash.len, 16)],
&name_buf,
);
if (parsed.list) {
for (members.items) |m| streams.out.print("{s}/{s}\n", .{ top, m.path }) catch {};
return .ok;
}
// Members carry the CHECK-IN's own time, never the clock: it is what makes
// two runs on two machines agree, and `SOURCE_DATE_EPOCH` already pins it
// at the moment the D card was written.
const mtime_ms = repo.state().mtimeOf(rid) orelse 0;
const mtime: u64 = mtime_ms / 1000;
return writeArchive(kind, gpa, io, &repo, streams, verb, parsed.output.?, top, members.items, mtime);
}
/// Whether an output path means "write nothing" — fossil's convention, and the
/// half of `--list` that lets a preview run against a real command line.
fn writesNothing(path: []const u8) bool {
return path.len == 0 or std.mem.eql(u8, path, "/dev/null");
}
fn writeArchive(
kind: Kind,
gpa: std.mem.Allocator,
io: std.Io,
repo: *Repository,
streams: cli.Streams,
verb: []const u8,
output: []const u8,
top: []const u8,
members: []const Member,
mtime: u64,
) cli.Exit {
if (writesNothing(output)) return .ok;
var out_file = std.Io.Dir.cwd().createFile(io, output, .{ .truncate = true }) catch {
streams.err.print("zit {s}: cannot create '{s}'\n", .{ verb, output }) catch {};
return .failure;
};
defer out_file.close(io);
var file_buf: [64 * 1024]u8 = undefined;
var file_writer = out_file.writerStreaming(io, &file_buf);
return switch (kind) {
.tarball => writeTar(gpa, repo, streams, verb, output, top, members, mtime, &file_writer),
.zip => writeZip(gpa, repo, streams, verb, output, top, members, mtime, &file_writer),
};
}
fn writeTar(
gpa: std.mem.Allocator,
repo: *Repository,
streams: cli.Streams,
verb: []const u8,
output: []const u8,
top: []const u8,
members: []const Member,
mtime: u64,
file_writer: *std.Io.File.Writer,
) cli.Exit {
// gzip through the shared deflate: its container header is a fixed
// constant with a zeroed mtime and no host byte, so the compressed wrapper
// cannot leak a build environment into an otherwise reproducible archive.
var window: [std.compress.flate.max_window_len]u8 = undefined;
var gz = std.compress.flate.Compress.init(
&file_writer.interface,
&window,
.gzip,
.default,
) catch return writeFail(streams, verb, output);
const w = &gz.writer;
for (members) |m| {
const bytes = (repo.blob(gpa, m.hash) catch |e| return openFail(streams, verb, e)) orelse {
// Presence was checked above, so reaching here means the store
// changed under us mid-run. Fail rather than emit a short member.
streams.err.print("zit {s}: content for '{s}' disappeared mid-archive\n", .{ verb, m.path }) catch {};
return .failure;
};
defer gpa.free(bytes);
if (bytes.len > max_member_bytes)
return fail(streams, verb, "'{s}' is larger than one archive member may be", .{m.path});
// A symlink's blob content IS its target path, so it becomes the link
// field of a symlink header and contributes no member content — a
// writer that missed this would emit a small text file where a link
// belonged, and the archive would look perfectly correct until use.
const is_link = m.mode == .symlink;
const size: u64 = if (is_link) 0 else bytes.len;
const link_target: ?[]const u8 = if (is_link) bytes else null;
if (is_link and bytes.len > archive.ustar_name_len) {
return fail(streams, verb, "'{s}' links to a target too long for tar", .{m.path});
}
var path_buf: [zit.managed.max_path_len + archive.max_top_dir_len + 1]u8 = undefined;
const joined = std.fmt.bufPrint(&path_buf, "{s}/{s}", .{ top, m.path }) catch
return fail(streams, verb, "'{s}' is too long to archive under '{s}'", .{ m.path, top });
archive.beginTarMember(w, joined, size, m.mode, mtime, link_target) catch
return writeFail(streams, verb, output);
if (!is_link) {
w.writeAll(bytes) catch return writeFail(streams, verb, output);
archive.endTarMember(w, size) catch return writeFail(streams, verb, output);
}
}
archive.writeTarEnd(w) catch return writeFail(streams, verb, output);
gz.finish() catch return writeFail(streams, verb, output);
file_writer.interface.flush() catch return writeFail(streams, verb, output);
return .ok;
}
/// The zip writer. Structurally different from tar for one reason: zip's
/// header carries the compressed size and the CRC, both of which are only
/// known after compressing, so each member is deflated into memory before its
/// header is written. The alternative — a trailing data descriptor — is what a
/// streaming writer uses and is still mishandled by a meaningful minority of
/// readers; blobs are already reconstructed whole, so the compatible choice
/// costs nothing here.
fn writeZip(
gpa: std.mem.Allocator,
repo: *Repository,
streams: cli.Streams,
verb: []const u8,
output: []const u8,
top: []const u8,
members: []const Member,
mtime: u64,
file_writer: *std.Io.File.Writer,
) cli.Exit {
const w = &file_writer.interface;
var entries: std.ArrayListUnmanaged(archive.ZipMember) = .empty;
defer {
for (entries.items) |e| gpa.free(e.path);
entries.deinit(gpa);
}
var offset: u32 = 0;
for (members) |m| {
const bytes = (repo.blob(gpa, m.hash) catch |e| return openFail(streams, verb, e)) orelse {
streams.err.print("zit {s}: content for '{s}' disappeared mid-archive\n", .{ verb, m.path }) catch {};
return .failure;
};
defer gpa.free(bytes);
if (bytes.len > archive.zip_max_member_bytes)
return fail(streams, verb, "'{s}' needs zip64, which this writer does not emit", .{m.path});
const joined = std.fmt.allocPrint(gpa, "{s}/{s}", .{ top, m.path }) catch return oom(streams, verb);
var joined_owned = true;
defer if (joined_owned) gpa.free(joined);
if (joined.len > std.math.maxInt(u16))
return fail(streams, verb, "'{s}' is too long to name in a zip", .{m.path});
// A symlink's CONTENT is its target path — that is what a reader
// recreates the link from, given S_IFLNK in the external attributes.
// Stored rather than deflated: a target is a few dozen bytes, and
// deflate would make it bigger.
const is_link = m.mode == .symlink;
const crc = zit.crc32.Crc32.hash(bytes);
var payload: []const u8 = bytes;
var owned_payload = false;
var method: u16 = archive.zip_method_stored;
if (!is_link and bytes.len != 0) {
const deflated = deflateRaw(gpa, bytes) catch return oom(streams, verb);
// Only take the compressed form when it actually helps: an
// already-compressed blob deflates LARGER, and storing it then is
// both smaller and faster to read.
if (deflated.len < bytes.len) {
payload = deflated;
owned_payload = true;
method = archive.zip_method_deflate;
} else {
gpa.free(deflated);
}
}
defer if (owned_payload) gpa.free(payload);
const entry: archive.ZipMember = .{
.path = joined,
.mode = m.mode,
.crc = crc,
.compressed_size = @intCast(payload.len),
.uncompressed_size = @intCast(bytes.len),
.method = method,
.local_offset = offset,
};
archive.writeZipLocal(w, entry, mtime) catch return writeFail(streams, verb, output);
w.writeAll(payload) catch return writeFail(streams, verb, output);
offset += archive.zipLocalHeaderLen(joined) + @as(u32, @intCast(payload.len));
entries.append(gpa, entry) catch return oom(streams, verb);
joined_owned = false; // the entry list owns it now
}
// The central directory: the record a reader trusts for metadata, and the
// only place a member's mode appears.
const cd_offset = offset;
var cd_size: u32 = 0;
for (entries.items) |e| {
archive.writeZipCentral(w, e, mtime) catch return writeFail(streams, verb, output);
cd_size += archive.zipCentralHeaderLen(e.path);
}
if (entries.items.len > std.math.maxInt(u16))
return fail(streams, verb, "more members than a zip32 central directory can count", .{});
archive.writeZipEnd(w, @intCast(entries.items.len), cd_size, cd_offset) catch
return writeFail(streams, verb, output);
file_writer.interface.flush() catch return writeFail(streams, verb, output);
return .ok;
}
/// Deflate `bytes` into a fresh buffer with no container framing — zip carries
/// the checksum and the sizes in its own headers, so the raw bit stream is
/// exactly what a member holds. The level is pinned, so the output is a pure
/// function of the input and two runs agree.
fn deflateRaw(gpa: std.mem.Allocator, bytes: []const u8) error{OutOfMemory}![]u8 {
// `Compress.init` asserts its OUTPUT has a buffer of more than 8 bytes, and
// a freshly-`init`ed Allocating writer has none — so the capacity here is a
// precondition, not a performance hint.
var aw = try std.Io.Writer.Allocating.initCapacity(gpa, @max(64, bytes.len / 2));
errdefer aw.deinit();
const window = try gpa.alloc(u8, std.compress.flate.max_window_len);
defer gpa.free(window);
var c = std.compress.flate.Compress.init(&aw.writer, window, .raw, .default) catch
return error.OutOfMemory;
c.writer.writeAll(bytes) catch return error.OutOfMemory;
c.finish() catch return error.OutOfMemory;
return aw.toOwnedSlice() catch error.OutOfMemory;
}
pub fn runZip(
gpa: std.mem.Allocator,
io: std.Io,
arena: std.mem.Allocator,
verb_args: []const []const u8,
streams: cli.Streams,
amb: cli.Ambient,
) cli.Exit {
return run(.zip, gpa, io, arena, verb_args, streams, amb);
}
fn fail(s: cli.Streams, verb: []const u8, comptime fmt: []const u8, args: anytype) cli.Exit {
s.err.print("zit {s}: ", .{verb}) catch {};
s.err.print(fmt ++ "\n", args) catch {};
return .failure;
}
fn oom(s: cli.Streams, verb: []const u8) cli.Exit {
s.err.print("zit {s}: out of memory\n", .{verb}) catch {};
return .failure;
}
fn writeFail(s: cli.Streams, verb: []const u8, output: []const u8) cli.Exit {
s.err.print("zit {s}: cannot write '{s}'\n", .{ verb, output }) catch {};
return .failure;
}
fn admitFail(s: cli.Streams, verb: []const u8, path: []const u8, e: archive.Error) cli.Exit {
switch (e) {
error.ReservedPath => s.err.print(
"zit {s}: the check-in names a reserved control path '{s}'; refusing to archive it\n",
.{ verb, path },
) catch {},
error.PathTooLong => s.err.print("zit {s}: path '{s}' is too long\n", .{ verb, path }) catch {},
}
return .failure;
}
fn filesFail(s: cli.Streams, verb: []const u8, e: Repository.FilesError) cli.Exit {
switch (e) {
error.OutOfMemory => return oom(s, verb),
error.NotARepo => s.err.writeAll(not_a_repo_msg) catch {},
error.CannotRead => s.err.print("zit {s}: cannot read repository: input/output error\n", .{verb}) catch {},
error.Corrupt => s.err.print("zit {s}: cannot read repository: repository is corrupt\n", .{verb}) catch {},
error.NotManifest => s.err.print("zit {s}: not a check-in\n", .{verb}) catch {},
error.BaselineUnresolvable => s.err.print("zit {s}: cannot resolve the baseline of a delta check-in\n", .{verb}) catch {},
}
return .failure;
}
fn openFail(s: cli.Streams, verb: []const u8, e: Repository.OpenError) cli.Exit {
switch (e) {
error.OutOfMemory => return oom(s, verb),
error.NotARepo => s.err.writeAll(not_a_repo_msg) catch {},
error.CannotRead => s.err.print("zit {s}: cannot read repository: input/output error\n", .{verb}) catch {},
error.Corrupt => s.err.print("zit {s}: cannot read repository: repository is corrupt\n", .{verb}) catch {},
}
return .failure;
}