Artifact [96d0e1157a]

Artifact 96d0e1157a9555a2e21bbbbe05ab5e49b899ff3f9a7f248d08f29e6266e979b4:

• File src/sim/faulty_file.zig — part of check-in [d6cf7bf59c] at 2026-09-28 11:59:39 on branch trunk — Lock the store with exclusive-create lock files instead of flock

  Writers create .zit.lock with O_CREAT|O_EXCL before opening the store. A waiter names the holder and gives up after a minute; a dead holder's lock is broken; pack and shun --force hold the writer lock across their final check and rename. Release 0.3.33. (user: dmestas, size: 15771) [annotate] [blame] [check-ins using]

//! Deterministic storage fault simulator (storage-track-plan.md §B). One
//! `StorageFile` implementation whose every fault is a draw from the run's
//! `std.Random` — the seed is still the whole universe, this just gives the
//! seed a device to lie through.
//!
//! Native design, no RFC section: storage-track-plan.md §B plus the four
//! crash-recovery invariants are the oracle, checked by
//! sim/storage_sweep.zig's DST sweep.
//!
//! The read-fault dimension (`read_rot_pct`: latent sector errors surfacing
//! at read time on a store that opened clean) follows TigerBeetle's
//! storage-fault-injection lineage — its simulator models bit rot that a
//! read must detect, never silently serve.

const std = @import("std");
const storage = @import("zit").storage;

/// Per-fault trigger probabilities, each a percent in `0..=100` drawn
/// against fresh bits from the run PRNG. All zero (the default) is a
/// perfectly healthy disk — DST phase 2 "heals" by resetting to this.
pub const FaultCfg = struct {
    /// append() persists a strict prefix (`k < n`) and returns error.Io.
    short_write_pct: u8 = 0,
    /// sync() reports success without moving `durable_len` — the next
    /// crash then drops everything it claimed to have saved.
    lost_tail_sync_pct: u8 = 0,
    /// On crashRestart, force the cut exactly at `durable_len` (tear the
    /// whole unsynced tail) instead of a random point within it.
    crash_cut_at_durable_pct: u8 = 0,
    /// On crashRestart, additionally seed a few bytes before the cut as
    /// corrupt (payload-derived bitflip, never self-heals).
    crash_rot_pct: u8 = 0,
    /// Latent sector error: on a read of an INTACT byte, with this
    /// probability intern one offset within the read span as permanently
    /// corrupt (payload-derived mask, never self-heals) — bit rot surfacing
    /// at read time on a store that opened clean, independent of any crash.
    /// The distinct dimension from `crash_rot_pct`, which only fires at a
    /// crash boundary.
    read_rot_pct: u8 = 0,
    /// When the cut isn't forced to `durable_len`, sometimes round it down
    /// to a sector boundary (torn mid-sector writes are the common case on
    /// real disks).
    sector_round: u64 = 512,
    sector_round_pct: u8 = 0,
};

/// One `StorageFile` implementation over an in-memory byte buffer, standing
/// in for a real disk that can tear writes, lie about fsync, and flip bits.
pub const FaultyFile = struct {
    gpa: std.mem.Allocator,
    random: std.Random,
    bytes: std.ArrayList(u8),
    /// Watermark: bytes in `[0, durable_len)` survived the last true fsync.
    /// Monotonic non-decreasing for the life of the file — sync only moves
    /// it forward, and a crash cut is always chosen `>= durable_len`.
    durable_len: u64,
    /// Sorted, deduped offsets that read back flipped. Membership is
    /// permanent once set — the corresponding fault never self-heals.
    corrupt: std.ArrayList(u64),
    cfg: FaultCfg,

    /// Explicit upper bound on the simulated file (storage-track-plan.md
    /// §B): keeps the sweep's growth bounded and provable, not merely
    /// convenient.
    pub const max_bytes: u64 = 64 << 20;

    pub fn init(gpa: std.mem.Allocator, random: std.Random) FaultyFile {
        return .{
            .gpa = gpa,
            .random = random,
            .bytes = .empty,
            .durable_len = 0,
            .corrupt = .empty,
            .cfg = .{},
        };
    }

    pub fn deinit(self: *FaultyFile) void {
        self.bytes.deinit(self.gpa);
        self.corrupt.deinit(self.gpa);
        self.* = undefined;
    }

    pub fn storageFile(self: *FaultyFile) storage.StorageFile {
        return .{ .context = self, .vtable = &vtable };
    }

    /// Simulate power loss and reopen: pick a cut in `[durable_len, len]`
    /// (weighted toward exactly `durable_len`, sometimes sector-rounded),
    /// drop everything past it, and maybe rot a few bytes before it.
    pub fn crashRestart(self: *FaultyFile) void {
        std.debug.assert(self.durable_len <= self.bytes.items.len);
        const unsynced = self.bytes.items.len - self.durable_len;
        var cut: u64 = self.durable_len;
        if (unsynced > 0 and !drawPct(self.random, self.cfg.crash_cut_at_durable_pct)) {
            cut += self.random.uintAtMost(u64, unsynced);
            if (cut > self.durable_len and self.cfg.sector_round != 0 and
                drawPct(self.random, self.cfg.sector_round_pct))
            {
                const rounded = (cut / self.cfg.sector_round) * self.cfg.sector_round;
                cut = @max(rounded, self.durable_len);
            }
        }
        std.debug.assert(cut >= self.durable_len and cut <= self.bytes.items.len);

        if (cut > 0 and drawPct(self.random, self.cfg.crash_rot_pct)) {
            const budget = @min(cut, 4);
            const count = 1 + self.random.uintLessThan(u64, budget);
            var i: u64 = 0;
            while (i < count) : (i += 1) self.addCorrupt(self.random.uintLessThan(u64, cut));
        }

        self.bytes.shrinkRetainingCapacity(@intCast(cut));
        self.durable_len = cut;
        self.pruneCorruptAbove(cut);
        std.debug.assert(self.durable_len <= self.bytes.items.len);
        std.debug.assert(self.bytes.items.len <= max_bytes);
    }

    fn addCorrupt(self: *FaultyFile, off: u64) void {
        std.debug.assert(off < self.bytes.items.len);
        for (self.corrupt.items) |existing| {
            if (existing == off) return;
        }
        // Best-effort: missing one injected fault under real allocation
        // pressure only means slightly less fault coverage this round,
        // never a correctness gap in the store under test.
        self.corrupt.append(self.gpa, off) catch return;
        std.mem.sort(u64, self.corrupt.items, {}, std.sort.asc(u64));
    }

    fn pruneCorruptAbove(self: *FaultyFile, cut: u64) void {
        var write: usize = 0;
        for (self.corrupt.items) |off| {
            if (off < cut) {
                self.corrupt.items[write] = off;
                write += 1;
            }
        }
        self.corrupt.shrinkRetainingCapacity(write);
    }

    const vtable: storage.StorageFile.VTable = .{
        .append = appendImpl,
        .pread = preadImpl,
        .sync = syncImpl,
        .size = sizeImpl,
        .truncate = truncateImpl,
        .refresh_end = refreshEndImpl,
    };

    /// One writer by construction, so there is no other writer to exclude and
    /// no other writer's appends to re-read the length for.
    fn refreshEndImpl(_: *anyopaque) storage.StorageError!void {}

    fn appendImpl(ctx: *anyopaque, data: []const u8) storage.StorageError!void {
        const self: *FaultyFile = @ptrCast(@alignCast(ctx));
        std.debug.assert(self.bytes.items.len + data.len <= max_bytes);
        if (data.len > 0 and drawPct(self.random, self.cfg.short_write_pct)) {
            // k < n by construction: a "short write" that persisted every
            // requested byte would not be short.
            const k = self.random.uintLessThan(usize, data.len);
            self.bytes.appendSlice(self.gpa, data[0..k]) catch return error.Io;
            return error.Io;
        }
        self.bytes.appendSlice(self.gpa, data) catch return error.Io;
    }

    fn preadImpl(ctx: *anyopaque, offset: u64, buf: []u8) storage.StorageError!usize {
        const self: *FaultyFile = @ptrCast(@alignCast(ctx));
        // StorageFile contract: an offset at or past EOF reads 0 bytes — it
        // is neither an error nor a precondition violation, so the sim must
        // not assert a stricter bound than the capability it stands in for.
        if (offset >= self.bytes.items.len) return 0;
        const avail = self.bytes.items.len - offset;
        const n = @min(avail, buf.len);
        if (n == 0) return 0;
        const start: usize = @intCast(offset);
        @memcpy(buf[0..n], self.bytes.items[start..][0..n]);
        // Latent sector error: intern one offset in this read span as
        // permanently corrupt, so it flips here AND on every later read of
        // it (the `crash_rot`/`corruptInPlace` discipline: payload-derived,
        // never self-healing). Drawn only when the knob is armed — a healthy
        // disk (`read_rot_pct == 0`) draws nothing, so an existing sweep's
        // PRNG stream is byte-identical.
        if (drawPct(self.random, self.cfg.read_rot_pct)) {
            self.addCorrupt(offset + self.random.uintLessThan(u64, n));
        }
        for (self.corrupt.items) |off| {
            if (off >= offset and off < offset + n) {
                buf[@intCast(off - offset)] ^= corruptionMask(off);
            }
        }
        return n;
    }

    fn syncImpl(ctx: *anyopaque) storage.StorageError!void {
        const self: *FaultyFile = @ptrCast(@alignCast(ctx));
        if (drawPct(self.random, self.cfg.lost_tail_sync_pct)) return; // lies: durable_len unmoved
        self.durable_len = self.bytes.items.len;
    }

    fn sizeImpl(ctx: *anyopaque) storage.StorageError!u64 {
        const self: *FaultyFile = @ptrCast(@alignCast(ctx));
        return self.bytes.items.len;
    }

    fn truncateImpl(ctx: *anyopaque, len: u64) storage.StorageError!void {
        const self: *FaultyFile = @ptrCast(@alignCast(ctx));
        std.debug.assert(len <= max_bytes);
        const len_usize: usize = @intCast(len);
        if (len_usize <= self.bytes.items.len) {
            self.bytes.shrinkRetainingCapacity(len_usize);
        } else {
            const old = self.bytes.items.len;
            self.bytes.resize(self.gpa, len_usize) catch return error.Io;
            @memset(self.bytes.items[old..], 0);
        }
        self.pruneCorruptAbove(len);
        self.durable_len = @min(self.durable_len, len);
    }
};

fn drawPct(random: std.Random, pct: u8) bool {
    std.debug.assert(pct <= 100);
    if (pct == 0) return false;
    if (pct == 100) return true;
    return random.uintLessThan(u8, 100) < pct;
}

/// Payload-derived, pure function of the offset: every retry corrupts
/// identically (storage-track-plan.md §B). Always nonzero, so it always
/// actually flips something.
fn corruptionMask(off: u64) u8 {
    const h = std.hash.Wyhash.hash(0, std.mem.asBytes(&off));
    return @truncate(1 | (h & 7));
}

test "faulty_file §B: pread at or past EOF reads 0 bytes (StorageFile contract)" {
    var prng = std.Random.DefaultPrng.init(6);
    var faulty = FaultyFile.init(std.testing.allocator, prng.random());
    defer faulty.deinit();

    const file = faulty.storageFile();
    try file.append("ABCD");
    try file.sync();

    var buf: [4]u8 = undefined;
    try std.testing.expectEqual(@as(usize, 0), try file.pread(4, &buf)); // exactly at EOF
    try std.testing.expectEqual(@as(usize, 0), try file.pread(99, &buf)); // well past EOF
}

test "faulty_file §B: short write persists a k<n prefix and returns error.Io" {
    var prng = std.Random.DefaultPrng.init(1);
    var faulty = FaultyFile.init(std.testing.allocator, prng.random());
    defer faulty.deinit();
    faulty.cfg.short_write_pct = 100;

    const file = faulty.storageFile();
    try std.testing.expectError(error.Io, file.append("12345678"));
    const size = try file.size();
    try std.testing.expect(size < 8);

    var buf: [8]u8 = undefined;
    const n = try file.pread(0, buf[0..@intCast(size)]);
    try std.testing.expectEqual(size, n);
    try std.testing.expectEqualStrings("12345678"[0..@intCast(size)], buf[0..n]);
}

test "faulty_file §B: crash-restart tears the unsynced tail within [durable_len, size]" {
    var prng = std.Random.DefaultPrng.init(2);
    var faulty = FaultyFile.init(std.testing.allocator, prng.random());
    defer faulty.deinit();
    faulty.cfg.crash_rot_pct = 0; // isolate the tear from corruption

    const file = faulty.storageFile();
    try file.append("AAAA");
    try file.sync();
    try file.append("BBBB");
    faulty.crashRestart();

    const size = try file.size();
    try std.testing.expect(size >= 4 and size <= 8);
    var buf: [4]u8 = undefined;
    _ = try file.pread(0, &buf);
    try std.testing.expectEqualStrings("AAAA", &buf);
}

test "faulty_file §B: fsync-lost-tail drops every unsynced byte on the next crash" {
    var prng = std.Random.DefaultPrng.init(3);
    var faulty = FaultyFile.init(std.testing.allocator, prng.random());
    defer faulty.deinit();

    const file = faulty.storageFile();
    try file.append("AAAA");
    try file.sync();
    try file.append("BBBB");
    faulty.cfg.lost_tail_sync_pct = 100;
    try file.sync(); // lies: claims success, durable_len stays at 4
    faulty.cfg.crash_cut_at_durable_pct = 100;
    faulty.crashRestart();

    try std.testing.expectEqual(@as(u64, 4), try file.size());
}

test "faulty_file §B: a bitflip is payload-derived and never self-heals on retry" {
    var prng = std.Random.DefaultPrng.init(4);
    var faulty = FaultyFile.init(std.testing.allocator, prng.random());
    defer faulty.deinit();

    const file = faulty.storageFile();
    try file.append("AAAAAAAA");
    try file.sync();
    faulty.cfg.crash_cut_at_durable_pct = 100; // cut lands at len 8, nothing torn
    faulty.cfg.crash_rot_pct = 100; // force at least one corrupt offset
    faulty.crashRestart();
    try std.testing.expect(faulty.corrupt.items.len > 0);
    const off = faulty.corrupt.items[0];

    var first: [1]u8 = undefined;
    var second: [1]u8 = undefined;
    _ = try file.pread(off, &first);
    _ = try file.pread(off, &second);
    try std.testing.expectEqual(first[0], second[0]);
    try std.testing.expect(first[0] != 'A');
}

test "faulty_file §B: a read fault returns altered bytes without signalling an error" {
    var prng = std.Random.DefaultPrng.init(5);
    var faulty = FaultyFile.init(std.testing.allocator, prng.random());
    defer faulty.deinit();

    const file = faulty.storageFile();
    try file.append("ZZZZZZZZ");
    try file.sync();
    faulty.cfg.crash_cut_at_durable_pct = 100;
    faulty.cfg.crash_rot_pct = 100;
    faulty.crashRestart();
    try std.testing.expect(faulty.corrupt.items.len > 0);
    const off = faulty.corrupt.items[0];

    var buf: [1]u8 = undefined;
    const n = try file.pread(off, &buf); // no error -- a read fault, not a device fault
    try std.testing.expectEqual(@as(usize, 1), n);
    try std.testing.expect(buf[0] != 'Z');
}

test "faulty_file §B: read_rot flips an intact byte at read time and never self-heals" {
    var prng = std.Random.DefaultPrng.init(7);
    var faulty = FaultyFile.init(std.testing.allocator, prng.random());
    defer faulty.deinit();

    const file = faulty.storageFile();
    try file.append("ABCDEFGH");
    try file.sync(); // fully durable, no crash -- the store "opened clean"
    faulty.cfg.read_rot_pct = 100; // latent sector error engages on the next read
    std.debug.assert(faulty.corrupt.items.len == 0);

    var buf: [8]u8 = undefined;
    const n = try file.pread(0, &buf); // no error surfaced -- rot is served, not signalled
    try std.testing.expectEqual(@as(usize, 8), n);
    try std.testing.expect(!std.mem.eql(u8, "ABCDEFGH", &buf)); // at least one byte flipped
    try std.testing.expect(faulty.corrupt.items.len > 0); // interned permanently
    const off = faulty.corrupt.items[0];

    // The same offset reads back identically corrupt with the knob disarmed:
    // payload-derived and permanent, never self-healing on a later read.
    faulty.cfg.read_rot_pct = 0;
    var a: [1]u8 = undefined;
    var b: [1]u8 = undefined;
    _ = try file.pread(off, &a);
    _ = try file.pread(off, &b);
    try std.testing.expectEqual(a[0], b[0]);
    try std.testing.expect(a[0] != "ABCDEFGH"[@intCast(off)]);
}