//! The Fossil zlib container codec — the single implementation of the §4.1
//! compressed container shared by every layer that stores or transmits one.
//!
//! draft-fossil-sync-protocol-00 §4.1 L889-919: a container is a 4-byte
//! big-endian pre-compression input length followed by a zlib stream [RFC1950]
//! (2-byte header, DEFLATE body [RFC1951], 4-byte Adler-32 trailer). It is
//! neither raw DEFLATE nor gzip. Ordinary compression uses the zlib default
//! level; the inline body-login concatenation compresses at DEFLATE level 9;
//! receivers MUST be level-independent (§4.1 L916-918). The 4-byte field is
//! an unsigned 32-bit output-capacity value: a decode that finishes within
//! that capacity is accepted and returns the actual decoded bytes, which may
//! be fewer than declared; "bytes after a successfully decoded zlib stream are
//! ignored." The artifact hash names the UNCOMPRESSED bytes (sync §7.7 L1484),
//! so storage compression never affects an artifact's address.
//!
//! This is a LEAF module: it imports only `std` and sits below `logstore`,
//! `repostate`, and `sync`, which each previously carried their own cited copy
//! of this framing. It is the sole home for the §4.1 codec so the container
//! rules can only ever be fixed in one place.
const std = @import("std");
const flate = std.compress.flate;
/// The inflate side is the tree's own copy of the standard decoder with one
/// underflow fixed; see src/flate/Decompress.zig. Compression stays standard.
const Decompress = @import("flate/Decompress.zig");
/// §4.1 L916-918 — the two encode levels the protocol names: the zlib
/// default for ordinary single-input containers, and DEFLATE level 9 for the
/// inline body-login concatenation. Decode is level-independent, so this
/// choice never crosses the wire beyond the bytes it produces.
pub const Level = enum { default, best };
fn compressOptions(level: Level) flate.Compress.Options {
return switch (level) {
.default => .level_6,
.best => .level_9,
};
}
/// `Compress.init` asserts its output writer already has more than 8 bytes of
/// buffer capacity; `Writer.Allocating` starts empty, so the encoder pre-sizes
/// before calling it. 64 is a cited headroom over that `>8` floor, comfortably
/// covering the 4-byte length prefix plus the 2-byte zlib header written first.
const compress_out_min: usize = 64;
/// A container's length field is an unsigned 32-bit value (§4.1 L909-911), so
/// input longer than this cannot be framed and is rejected before any work.
pub const EncodeError = error{ OutOfMemory, InputTooLarge };
/// Every §4.1 decode-side rejection: a header shorter than the 4-byte length
/// prefix, a declared capacity over the caller's `max_output` bound or a stream
/// that overruns its declared capacity (both `OutputTooLarge`), and a malformed
/// zlib stream (`InvalidStream`).
pub const DecodeError = error{ OutOfMemory, TruncatedHeader, InvalidStream, OutputTooLarge };
/// §4.1 L889-892, L916-918 — wrap `bytes` in the §4.1 container: a 4-byte
/// big-endian pre-compression length, then an RFC1950 zlib stream at `level`.
/// The sender MUST encode the actual input length (§4.1 L892), which the
/// postcondition below asserts.
pub fn encode(gpa: std.mem.Allocator, bytes: []const u8, level: Level) EncodeError![]u8 {
if (bytes.len > std.math.maxInt(u32)) return error.InputTooLarge;
var aw = std.Io.Writer.Allocating.init(gpa);
defer aw.deinit();
aw.ensureUnusedCapacity(compress_out_min) catch return error.OutOfMemory;
aw.writer.writeInt(u32, @intCast(bytes.len), .big) catch return error.OutOfMemory;
const window = gpa.alloc(u8, flate.max_window_len) catch return error.OutOfMemory;
defer gpa.free(window);
var c = flate.Compress.init(&aw.writer, window, .zlib, compressOptions(level)) catch return error.OutOfMemory;
c.writer.writeAll(bytes) catch return error.OutOfMemory;
c.finish() catch return error.OutOfMemory;
const out = try aw.toOwnedSlice();
std.debug.assert(std.mem.readInt(u32, out[0..4], .big) == bytes.len); // §4.1 L892 — actual length encoded
return out;
}
/// The first `want` plaintext bytes of a §4.1 container, or fewer when the
/// artifact itself is shorter. Inflates only far enough to produce them and
/// abandons the stream, so the cost is the zlib header plus one block rather
/// than the whole artifact.
///
/// This is a CLASSIFIER's read, never a content read: the bytes it returns are
/// a prefix, so nothing may hash them, store them, or hand them to a parser
/// that would treat a truncated artifact as a whole one. `decode` remains the
/// only way to obtain an artifact.
///
/// Truncation is why this cannot enforce `decode`'s capacity postcondition: a
/// stream stopped early has no declared-length identity to check against. It
/// still rejects a header too short to carry the length prefix, and still
/// bounds its own iteration the way `decode` does.
pub fn decodePrefix(gpa: std.mem.Allocator, wire: []const u8, want: usize) DecodeError![]u8 {
if (wire.len < 4) return error.TruncatedHeader;
const declared = std.mem.readInt(u32, wire[0..4], .big);
const cap: usize = @min(want, @as(usize, declared));
if (cap == 0) return gpa.alloc(u8, 0) catch return error.OutOfMemory;
var fr = std.Io.Reader.fixed(wire[4..]);
const window = gpa.alloc(u8, flate.max_window_len) catch return error.OutOfMemory;
defer gpa.free(window);
var dc = Decompress.init(&fr, .zlib, window);
var aw = std.Io.Writer.Allocating.init(gpa);
defer aw.deinit();
// Same pump and the same bounded-iteration argument as `decode`; only the
// ceiling differs, and a short read is a legitimate result here rather
// than the truncation `decode` must reject.
var total: usize = 0;
const max_iters: usize = wire.len + cap + 64;
var iters: usize = 0;
while (total < cap and iters < max_iters) : (iters += 1) {
const n = dc.reader.stream(&aw.writer, .limited(cap - total)) catch |err| switch (err) {
error.EndOfStream => break,
error.WriteFailed => return error.OutOfMemory,
error.ReadFailed => return error.InvalidStream,
};
total += n;
}
if (iters >= max_iters) return error.InvalidStream;
return aw.toOwnedSlice();
}
/// §4.1 L901-914 — decode a container produced by `encode`. The length field
/// is a capacity, not an exact length: a decode that finishes within that
/// capacity is accepted and returns the actual decoded byte count, which may be
/// less than declared. `max_output` bounds the declared length (rejecting a
/// lying header before any decompression work happens) and the actual stream is
/// pumped no further than `declared + 1` bytes, so a truncated-length lie
/// cannot drive unbounded allocation; the `+1` is what detects a stream that
/// overruns its declared capacity. Decode is compression-level-independent
/// (§4.1 L916-918).
pub fn decode(gpa: std.mem.Allocator, wire: []const u8, max_output: usize) DecodeError![]u8 {
if (wire.len < 4) return error.TruncatedHeader;
const declared = std.mem.readInt(u32, wire[0..4], .big);
if (declared > max_output) return error.OutputTooLarge;
var fr = std.Io.Reader.fixed(wire[4..]);
const window = gpa.alloc(u8, flate.max_window_len) catch return error.OutOfMemory;
defer gpa.free(window);
var dc = Decompress.init(&fr, .zlib, window);
var aw = std.Io.Writer.Allocating.init(gpa);
defer aw.deinit();
const cap: usize = @as(usize, declared) + 1;
var total: usize = 0;
// `stream()` may legitimately return 0 progress on a call that only does
// internal bookkeeping (e.g. parsing a block header) without yet emitting
// output bytes, so 0 is not itself a completion signal — only
// `error.EndOfStream` or reaching `cap` ends the pump. The iteration count
// is still explicitly bounded (input size plus the output cap plus
// headroom) so a stdlib regression that stalled forever would surface as
// `InvalidStream` rather than hang.
const max_iters: usize = wire.len + cap + 64;
var iters: usize = 0;
while (total < cap and iters < max_iters) : (iters += 1) {
const n = dc.reader.stream(&aw.writer, .limited(cap - total)) catch |err| switch (err) {
error.EndOfStream => break,
error.WriteFailed => return error.OutOfMemory,
error.ReadFailed => return error.InvalidStream,
};
total += n;
}
if (iters >= max_iters) return error.InvalidStream;
if (total > declared) return error.OutputTooLarge; // §4.1 L901-909: must fit within capacity
// A stream is finished only when its final block and zlib footer were
// read; the pump also stops on a capacity fill, which is not an end. And
// the decoder records the footer's Adler-32 without checking it, so the
// check is here: a damaged container can terminate early and cleanly (one
// flipped bit turned a block into a final block, in the nightly soak's
// finding) and would otherwise hand back wrong bytes as good ones.
if (dc.state != .end) return error.InvalidStream;
const out = try aw.toOwnedSlice();
errdefer gpa.free(out);
if (std.hash.Adler32.hash(out) != dc.container_metadata.zlib.adler) return error.InvalidStream;
std.debug.assert(out.len == total and out.len <= declared); // postcondition: accounting + capacity ceiling
return out;
}
test "decode: a container with one bit flipped is an error, never a crash" {
// The nightly fuzz soak's finding: a valid container (99 bytes declared)
// whose byte at offset 75 has one bit flipped drove the standard inflate
// into an underflow in its end-of-input bit peek; with that fixed, the
// same bytes decoded to 58 "good" bytes, because the flip made a block
// final and the zlib footer was never checked. Any peer can send this;
// the decoder must answer with an error, and the undamaged container
// must still decode.
const gpa = std.testing.allocator;
const hex =
"00000063789cb39a579c6d5b55d3a5bfe5b3986a8a4c6aa2715941d28dc955f93b36df4d2d1128cc8c9caad7ae95b5c2473b81b3dc647f68f1f7a2b6a2ecbc92d355f637e35efb6e9f50f920a148ceb1a860ee25e5982d498becd354d30bcba2955c554bab749af35257171703008eab2bb1";
var wire: [hex.len / 2]u8 = undefined;
_ = try std.fmt.hexToBytes(&wire, hex);
try std.testing.expectError(error.InvalidStream, decode(gpa, &wire, 1 << 20));
// The undamaged container still decodes.
var good = wire;
good[75] ^= 0x80;
const out = try decode(gpa, &good, 1 << 20);
defer gpa.free(out);
try std.testing.expectEqual(@as(usize, 0x63), out.len);
}
test "decode: garbage whose dynamic header runs off the end of the input is an error, never a crash" {
// The nightly fuzz soak's second finding in the inflate it vendors: 32
// random bytes whose dynamic-block header ends one byte short. The bit
// check before a code-length toss added the bits already consumed instead
// of subtracting them, so the toss advanced the reader past its end, the
// next peek computed a length of 2^64-1, and the read walked off the stack.
// The declared size is 287 MB: the cap must admit it, or `decode` refuses
// it up front and the inflate is never reached.
const gpa = std.testing.allocator;
const hex = "111bd49338079da3a6b91ea531ac63903f252e4d4c9def951e4c2fdca5761c85";
var wire: [hex.len / 2]u8 = undefined;
_ = try std.fmt.hexToBytes(&wire, hex);
if (decode(gpa, &wire, std.math.maxInt(u32))) |ok| {
gpa.free(ok);
return error.TestUnexpectedResult;
} else |_| {}
}