Artifact [79f2272cb4]

Artifact 79f2272cb492ea676fce456ae1e366b6aecae4d82b5c35ede39fe7a97c9f982e:

• File src/container.zig — part of check-in [7d26a31305] at 2026-08-05 18:16:19 on branch sub-manifest — pack: skip reconstructing artifacts that cannot be manifests

  deriveCandidateHints visited every artifact in the store and reconstructed each one in full, only for parseForImport to discard the blobs. Two bytes decide it instead: §4.4 makes byte 0 of a structural artifact a card letter and byte 1 its delimiter, so a prefix inflate classifies without paying for the whole artifact.

  Conservative both ways -- a delta frame or a prefix error falls through to the full read, and only a definite mismatch rejects -- so output is byte-identical, verified on every corpus tried.

  Pack of a full-form-dominated store (the shape import-git's automatic pack reads) 10.84s to 8.49s. Neutral on an already-packed store, where deltas dominate and cannot be classified this way.

  test 3495/3495, fmt. Issue #32. (user: dmestas, size: 8719) [annotate] [blame] [check-ins using]

//! The Fossil zlib container codec — the single implementation of the §4.1
//! compressed container shared by every layer that stores or transmits one.
//!
//! draft-fossil-sync-protocol-00 §4.1 L1192-1213: a container is a 4-byte
//! big-endian pre-compression input length followed by a zlib stream [RFC1950]
//! (2-byte header, DEFLATE body [RFC1951], 4-byte Adler-32 trailer). It is
//! neither raw DEFLATE nor gzip. Ordinary compression uses the zlib default
//! level; the inline body-login concatenation compresses at DEFLATE level 9;
//! receivers MUST be level-independent (§4.1 L1204-1206). The 4-byte field is
//! an unsigned 32-bit output-capacity value: a decode that finishes within
//! that capacity is accepted and returns the actual decoded bytes, which may
//! be fewer than declared; "bytes after a successfully decoded zlib stream are
//! ignored." The artifact hash names the UNCOMPRESSED bytes (§4.1 L1206-1208),
//! so storage compression never affects an artifact's address.
//!
//! This is a LEAF module: it imports only `std` and sits below `logstore`,
//! `repostate`, and `sync`, which each previously carried their own cited copy
//! of this framing. It is the sole home for the §4.1 codec so the container
//! rules can only ever be fixed in one place.

const std = @import("std");
const flate = std.compress.flate;

/// §4.1 L1204-1205 — the two encode levels the protocol names: the zlib
/// default for ordinary single-input containers, and DEFLATE level 9 for the
/// inline body-login concatenation. Decode is level-independent, so this
/// choice never crosses the wire beyond the bytes it produces.
pub const Level = enum { default, best };

fn compressOptions(level: Level) flate.Compress.Options {
    return switch (level) {
        .default => .level_6,
        .best => .level_9,
    };
}

/// `Compress.init` asserts its output writer already has more than 8 bytes of
/// buffer capacity; `Writer.Allocating` starts empty, so the encoder pre-sizes
/// before calling it. 64 is a cited headroom over that `>8` floor, comfortably
/// covering the 4-byte length prefix plus the 2-byte zlib header written first.
const compress_out_min: usize = 64;

/// A container's length field is an unsigned 32-bit value (§4.1 L1200-1201), so
/// input longer than this cannot be framed and is rejected before any work.
pub const EncodeError = error{ OutOfMemory, InputTooLarge };

/// Every §4.1 decode-side rejection: a header shorter than the 4-byte length
/// prefix, a declared capacity over the caller's `max_output` bound or a stream
/// that overruns its declared capacity (both `OutputTooLarge`), and a malformed
/// zlib stream (`InvalidStream`).
pub const DecodeError = error{ OutOfMemory, TruncatedHeader, InvalidStream, OutputTooLarge };

/// §4.1 L1192-1197, L1204-1205 — wrap `bytes` in the §4.1 container: a 4-byte
/// big-endian pre-compression length, then an RFC1950 zlib stream at `level`.
/// The sender MUST encode the actual input length (§4.1 L1199), which the
/// postcondition below asserts.
pub fn encode(gpa: std.mem.Allocator, bytes: []const u8, level: Level) EncodeError![]u8 {
    if (bytes.len > std.math.maxInt(u32)) return error.InputTooLarge;

    var aw = std.Io.Writer.Allocating.init(gpa);
    defer aw.deinit();
    aw.ensureUnusedCapacity(compress_out_min) catch return error.OutOfMemory;
    aw.writer.writeInt(u32, @intCast(bytes.len), .big) catch return error.OutOfMemory;

    const window = gpa.alloc(u8, flate.max_window_len) catch return error.OutOfMemory;
    defer gpa.free(window);

    var c = flate.Compress.init(&aw.writer, window, .zlib, compressOptions(level)) catch return error.OutOfMemory;
    c.writer.writeAll(bytes) catch return error.OutOfMemory;
    c.finish() catch return error.OutOfMemory;

    const out = try aw.toOwnedSlice();
    std.debug.assert(std.mem.readInt(u32, out[0..4], .big) == bytes.len); // §4.1 L1199 — actual length encoded
    return out;
}

/// §4.1 L1197-1213 — decode a container produced by `encode`. The length field
/// is a capacity, not an exact length: a decode that finishes within that
/// capacity is accepted and returns the actual decoded byte count, which may be
/// less than declared. `max_output` bounds the declared length (rejecting a
/// lying header before any decompression work happens) and the actual stream is
/// pumped no further than `declared + 1` bytes, so a truncated-length lie
/// cannot drive unbounded allocation; the `+1` is what detects a stream that
/// overruns its declared capacity. Decode is compression-level-independent
/// (§4.1 L1204-1206).
/// The first `want` plaintext bytes of a §4.1 container, or fewer when the
/// artifact itself is shorter. Inflates only far enough to produce them and
/// abandons the stream, so the cost is the zlib header plus one block rather
/// than the whole artifact.
///
/// This is a CLASSIFIER's read, never a content read: the bytes it returns are
/// a prefix, so nothing may hash them, store them, or hand them to a parser
/// that would treat a truncated artifact as a whole one. `decode` remains the
/// only way to obtain an artifact.
///
/// Truncation is why this cannot enforce `decode`'s capacity postcondition: a
/// stream stopped early has no declared-length identity to check against. It
/// still rejects a header too short to carry the length prefix, and still
/// bounds its own iteration the way `decode` does.
pub fn decodePrefix(gpa: std.mem.Allocator, wire: []const u8, want: usize) DecodeError![]u8 {
    if (wire.len < 4) return error.TruncatedHeader;
    const declared = std.mem.readInt(u32, wire[0..4], .big);
    const cap: usize = @min(want, @as(usize, declared));
    if (cap == 0) return gpa.alloc(u8, 0) catch return error.OutOfMemory;

    var fr = std.Io.Reader.fixed(wire[4..]);
    const window = gpa.alloc(u8, flate.max_window_len) catch return error.OutOfMemory;
    defer gpa.free(window);
    var dc = flate.Decompress.init(&fr, .zlib, window);

    var aw = std.Io.Writer.Allocating.init(gpa);
    defer aw.deinit();

    // Same pump and the same bounded-iteration argument as `decode`; only the
    // ceiling differs, and a short read is a legitimate result here rather
    // than the truncation `decode` must reject.
    var total: usize = 0;
    const max_iters: usize = wire.len + cap + 64;
    var iters: usize = 0;
    while (total < cap and iters < max_iters) : (iters += 1) {
        const n = dc.reader.stream(&aw.writer, .limited(cap - total)) catch |err| switch (err) {
            error.EndOfStream => break,
            error.WriteFailed => return error.OutOfMemory,
            error.ReadFailed => return error.InvalidStream,
        };
        total += n;
    }
    if (iters >= max_iters) return error.InvalidStream;
    return aw.toOwnedSlice();
}

pub fn decode(gpa: std.mem.Allocator, wire: []const u8, max_output: usize) DecodeError![]u8 {
    if (wire.len < 4) return error.TruncatedHeader;
    const declared = std.mem.readInt(u32, wire[0..4], .big);
    if (declared > max_output) return error.OutputTooLarge;

    var fr = std.Io.Reader.fixed(wire[4..]);
    const window = gpa.alloc(u8, flate.max_window_len) catch return error.OutOfMemory;
    defer gpa.free(window);
    var dc = flate.Decompress.init(&fr, .zlib, window);

    var aw = std.Io.Writer.Allocating.init(gpa);
    defer aw.deinit();

    const cap: usize = @as(usize, declared) + 1;
    var total: usize = 0;
    // `stream()` may legitimately return 0 progress on a call that only does
    // internal bookkeeping (e.g. parsing a block header) without yet emitting
    // output bytes, so 0 is not itself a completion signal — only
    // `error.EndOfStream` or reaching `cap` ends the pump. The iteration count
    // is still explicitly bounded (input size plus the output cap plus
    // headroom) so a stdlib regression that stalled forever would surface as
    // `InvalidStream` rather than hang.
    const max_iters: usize = wire.len + cap + 64;
    var iters: usize = 0;
    while (total < cap and iters < max_iters) : (iters += 1) {
        const n = dc.reader.stream(&aw.writer, .limited(cap - total)) catch |err| switch (err) {
            error.EndOfStream => break,
            error.WriteFailed => return error.OutOfMemory,
            error.ReadFailed => return error.InvalidStream,
        };
        total += n;
    }
    if (iters >= max_iters) return error.InvalidStream;

    if (total > declared) return error.OutputTooLarge; // §4.1 L1209-1210: must fit within capacity
    const out = try aw.toOwnedSlice();
    std.debug.assert(out.len == total and out.len <= declared); // postcondition: accounting + capacity ceiling
    return out;
}