Skip to content
zit Artifact [0916495aea]
Login

Artifact [0916495aea]

Check-ins Using Download Hex

Artifact 0916495aeac5301b39fd392dc42e7f5f4ce9ae6f99ef7a038f12e0d6a50d000e:

//! #174 — content classification, ported from fossil's `lookslike.c`.
//!
//! One question, asked in two places: which of the classes a repository may
//! declare a policy about does this file's content fall into? The commit gate
//! (`zit commit`) and the push gate (`sync.PushPolicy`) must answer it the
//! same way, or a file zit itself committed would be refused when pushed to a
//! zit server — so the answer lives here rather than in either caller.
//!
//! Fossil's originals are `could_be_utf16` / `starts_with_utf16_bom`,
//! `looks_like_utf16`, and `invalid_utf8` with its `lb_tab` lead-byte table.

const std = @import("std");

/// What one file's bytes look like, in the four classes a repository can
/// declare a policy about.
pub const Look = struct {
    /// The blob carries a UTF-16 byte-order mark and an even byte count.
    utf16: bool = false,
    /// A zero unit — a byte outside UTF-16, a 16-bit unit inside it. This is
    /// the distinction that stops a UTF-16 file from reading as binary.
    nul: bool = false,
    /// Text that is not valid UTF-8. Never set together with `nul`: fossil
    /// asks this question only of blobs it did not already call binary.
    invalid_utf8: bool = false,
    cr: bool = false,
};

/// fossil's `could_be_utf16` (lookslike.c): the blob's byte order when it
/// starts with a UTF-16 byte-order mark, else null. True is big-endian.
///
/// The three refusals are fossil's, not additions: an odd byte count cannot be
/// 16-bit units at all, a blob shorter than the mark cannot carry one, and
/// `FF FE 00 00` is the UTF-32 mark — `starts_with_utf16_bom` rejects a mark
/// whose next two bytes are both zero for exactly that reason.
fn utf16ByteOrder(content: []const u8) ?bool {
    if (content.len < 2 or content.len % 2 != 0) return null;
    if (content.len >= 4 and content[2] == 0 and content[3] == 0) return null;
    if (content[0] == 0xFE and content[1] == 0xFF) return true;
    if (content[0] == 0xFF and content[1] == 0xFE) return false;
    return null;
}

/// One file's content classes. UTF-16 is scanned as 16-bit units and every
/// other blob as bytes, which is the whole reason fossil has two scanners: the
/// zero bytes inside `a\0b\0` are halves of ASCII units, not NULs.
///
/// `cr` keeps zit's CR/LF-pair rule on the byte path (a lone CR is not yet a
/// class here); on the UTF-16 path it is fossil's `LOOK_CR` — any CR unit —
/// because that is the flag whose branch names both globs.
pub fn classify(content: []const u8) Look {
    if (utf16ByteOrder(content)) |big_endian| {
        std.debug.assert(content.len >= 2 and content.len % 2 == 0);
        var look: Look = .{ .utf16 = true };
        // Bounded by the blob: starts past the mark, steps two bytes, and
        // stops one unit short of the end.
        var i: usize = 2;
        while (i + 1 < content.len) : (i += 2) {
            const hi: u16 = if (big_endian) content[i] else content[i + 1];
            const lo: u16 = if (big_endian) content[i + 1] else content[i];
            const unit: u16 = (hi << 8) | lo;
            if (unit == 0) look.nul = true;
            if (unit == '\r') look.cr = true;
        }
        return look;
    }
    const nul = std.mem.indexOfScalar(u8, content, 0) != null;
    const look: Look = .{
        .nul = nul,
        // fossil asks the UTF-8 question only of blobs that are not binary
        // (checkin.c: `if( !(lookFlags & LOOK_BINARY) && invalid_utf8(...) )`).
        // Invalid UTF-8 is a class of TEXT files -- a PNG is binary, not badly
        // encoded, and it is `binary-glob` that has to be able to exempt it.
        .invalid_utf8 = !nul and invalidUtf8(content),
        .cr = std.mem.indexOf(u8, content, "\r\n") != null,
    };
    // Negative space: the byte path never claims UTF-16, so a caller checking
    // the encoding classes cannot be handed one derived from the wrong scan,
    // and no blob is ever in both the binary and the invalid-UTF-8 class.
    std.debug.assert(!look.utf16);
    std.debug.assert(!(look.nul and look.invalid_utf8));
    return look;
}

/// The valid range for the byte following one UTF-8 lead byte: `size` bytes
/// starting at `start`. A `size` of 0 means no byte is valid, i.e. this is not
/// a lead byte.
const Utf8Range = struct { start: u8, size: u8 };

/// fossil's `lb_tab` (lookslike.c), indexed by `lead - 0x80` rather than by
/// its doubled byte. The macros it is built from name their lead bytes: US2A
/// is 0xC0, US2B is 0xC2-0xDF, US3A is 0xE0, US3B is 0xE1-0xEF, US4A is 0xF0,
/// US4B is 0xF1-0xF3, US4C is 0xF4, and US0A is every other byte.
const lb_tab: [128]Utf8Range = blk: {
    const none: Utf8Range = .{ .start = 0x00, .size = 0x00 };
    var t: [128]Utf8Range = @splat(none);
    t[0xC0 - 0x80] = .{ .start = 0x80, .size = 0x01 };
    for (0xC2..0xE0) |c| t[c - 0x80] = .{ .start = 0x80, .size = 0x40 };
    t[0xE0 - 0x80] = .{ .start = 0xA0, .size = 0x20 };
    for (0xE1..0xF0) |c| t[c - 0x80] = .{ .start = 0x80, .size = 0x40 };
    t[0xF0 - 0x80] = .{ .start = 0x90, .size = 0x30 };
    for (0xF1..0xF4) |c| t[c - 0x80] = .{ .start = 0x80, .size = 0x40 };
    t[0xF4 - 0x80] = .{ .start = 0x80, .size = 0x10 };
    break :blk t;
};

/// fossil's `invalid_utf8` (lookslike.c), walked exactly as fossil walks it:
/// the lead byte is doubled in place (dropping its top bit), the table entry
/// at that index gives the range the next byte must land in, and the doubled
/// value either becomes the next pseudo-lead (`| 3`, one more continuation
/// expected) or a space (the sequence is complete).
///
/// It is deliberately laxer than a strict validator — fossil's own comment
/// admits the overlong NUL that Java and Tcl emit, and CESU-8 / WTF-8
/// surrogates. Porting it faithfully is the point: a zit that refused a file
/// fossil accepts would be a parity bug in the direction that costs a commit.
pub fn invalidUtf8(content: []const u8) bool {
    if (content.len == 0) return false; // empty file -> OK
    var c: u8 = content[0];
    var i: usize = 0;
    // `remaining` is `content.len - i` by construction: every arm below
    // advances `i` exactly once per turn, so the index stays in range.
    var remaining: usize = content.len;
    while (remaining > 1) : (remaining -= 1) {
        std.debug.assert(i + 1 < content.len);
        i += 1;
        if (c < 0x80) {
            c = content[i];
            continue;
        }
        c = c *% 2; // multiply by 2 and get rid of the highest bit
        const def = lb_tab[c / 2];
        // Wrapping: a byte below `start` wraps to at least 0x60, and no range
        // is wider than 0x40, so an under-range byte is never mistaken for an
        // in-range one. This is fossil's unsigned cast, in eight bits.
        if (content[i] -% def.start >= def.size) return true;
        c = if (c >= 0xC0) c | 3 else ' ';
    }
    return c >= 0x80; // the final lead byte must be ASCII
}

test "#174 item 6: classify reads UTF-16, NUL, invalid UTF-8 and CR as fossil's lookslike.c does" {
    // Little-endian BOM. The zero bytes are the high halves of ASCII units,
    // not NUL characters: fossil scans a UTF-16 blob as 16-bit units
    // (`looks_like_utf16`), which is why such a file is not "binary".
    const le = classify("\xFF\xFEa\x00b\x00");
    try std.testing.expect(le.utf16 and !le.nul and !le.cr and !le.invalid_utf8);

    // Big-endian BOM, same text, same answer.
    const be = classify("\xFE\xFF\x00a\x00b");
    try std.testing.expect(be.utf16 and !be.nul and !be.cr and !be.invalid_utf8);

    // A zero UNIT is a NUL, and binary wins over encoding downstream.
    const utf16_nul = classify("\xFF\xFEa\x00\x00\x00");
    try std.testing.expect(utf16_nul.utf16 and utf16_nul.nul);

    // A CR unit in a UTF-16 blob is a CR, exactly as fossil's LOOK_CR.
    const utf16_cr = classify("\xFF\xFE\x0D\x00\x0A\x00");
    try std.testing.expect(utf16_cr.utf16 and utf16_cr.cr and !utf16_cr.nul);

    // FF FE 00 00 is a UTF-32 byte-order mark, not UTF-16:
    // `starts_with_utf16_bom` rejects a BOM whose next two bytes are both
    // zero. It falls through to the byte scan, where the zeros are NULs.
    const utf32 = classify("\xFF\xFE\x00\x00a\x00");
    try std.testing.expect(!utf32.utf16 and utf32.nul);

    // An odd byte count cannot be UTF-16 (`could_be_utf16`), so the same
    // bytes plus one are read as a binary byte stream.
    const odd = classify("\xFF\xFEa\x00b");
    try std.testing.expect(!odd.utf16 and odd.nul);

    // Byte scan: the UTF-8 lead-byte table decides validity.
    try std.testing.expect(classify("\xC3\x28").invalid_utf8);
    try std.testing.expect(!classify("\xE2\x82\xAC").invalid_utf8);
    try std.testing.expect(classify("\xC0\xAF").invalid_utf8); // overlong '/'
    try std.testing.expect(classify("a\r\nb").cr);

    // Empty is clean in every class -- fossil's "empty file -> text".
    const empty = classify("");
    try std.testing.expect(!empty.utf16 and !empty.nul and !empty.cr and !empty.invalid_utf8);
}

test "#174 item 6: fossil's UTF-8 table accepts everything std.unicode does, and two things it does not" {
    // fossil's `invalid_utf8` is deliberately laxer than a strict validator:
    // its comment names the overlong NUL that Java and Tcl emit, and CESU-8 /
    // WTF-8 surrogates, as sequences it treats as valid. Porting the table
    // faithfully means inheriting that, so it is pinned here rather than
    // discovered later as a parity gap.
    try std.testing.expect(!invalidUtf8("\xC0\x80")); // overlong NUL
    try std.testing.expect(!std.unicode.utf8ValidateSlice("\xC0\x80"));
    try std.testing.expect(!invalidUtf8("\xED\xA0\x80")); // CESU-8 lead surrogate
    try std.testing.expect(!std.unicode.utf8ValidateSlice("\xED\xA0\x80"));

    // Everywhere else the direction that matters for a commit gate holds:
    // nothing std calls valid is ever refused. A seeded corpus of arbitrary
    // bytes finds no exception.
    var prng = std.Random.DefaultPrng.init(0x1741741717417417);
    const rand = prng.random();
    var buf: [24]u8 = undefined;
    var i: usize = 0;
    while (i < 20_000) : (i += 1) {
        const len = rand.uintLessThan(usize, buf.len + 1);
        const slice = buf[0..len];
        for (slice) |*b| b.* = rand.int(u8);
        if (std.unicode.utf8ValidateSlice(slice)) try std.testing.expect(!invalidUtf8(slice));
    }
}

test "#174 item 6: a blob with a NUL is binary, never invalid UTF-8" {
    // fossil computes `fHasInvalidUtf8` only when the blob is not binary
    // (checkin.c `commit_warning`: `if( !(lookFlags & LOOK_BINARY) &&
    // invalid_utf8(...) )`). Invalid UTF-8 is a class of TEXT files; a PNG is
    // binary, not badly encoded, and saying otherwise names the wrong class
    // and points the operator at the wrong glob.
    const png = "\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\xFF\xFE\x01";
    const look = classify(png);
    try std.testing.expect(look.nul);
    try std.testing.expect(!look.invalid_utf8);
    try std.testing.expect(!look.utf16);

    // The same bytes without the NUL are exactly the invalid-UTF-8 class.
    try std.testing.expect(classify("\x89PNG\r\n\x1a\nIHDR\xFF\xFE\x01").invalid_utf8);

    // A UTF-16 blob whose units really are zero is binary too, and the byte
    // path is never consulted for it, so it carries no UTF-8 verdict either.
    const wide_nul = classify("\xFF\xFE\x0D\x00\x00\x00");
    try std.testing.expect(wide_nul.utf16 and wide_nul.nul and !wide_nul.invalid_utf8);
}