Artifact [aa007dfb4e]

Artifact aa007dfb4e27b2546f730d7a66e9d9927165863c64f27c8c6ff1b12169931c8c:

• File src/sim/faulty_file.zig — part of check-in [5c445bb3f3] at 2026-09-28 02:39:03 on branch trunk — Take the writer lock inside LogStore.open and test the lock class

  LogStore.open now takes the exclusive lock before it reads the file's length and scans for the end, and a read-only handle refuses to append. purge undo opens the store through the shared writer path.

  A new DST sweep runs two to four writers interleaved against one log through a file that models the lock. It flags unlocked or misplaced appends as they happen, checks each writer sees everything committed before it locked and reads back its own writes, including deltas. A source check fails on any write-mode open of the store outside openStoreAppend. An e2e test races commits, tags and user-table writes on one store and counts what landed. (user: dmestas, size: 15752) [annotate] [blame] [check-ins using]

//! Deterministic storage fault simulator (storage-track-plan.md §B). One
//! `StorageFile` implementation whose every fault is a draw from the run's
//! `std.Random` — the seed is still the whole universe, this just gives the
//! seed a device to lie through.
//!
//! Native design, no RFC section: storage-track-plan.md §B plus the four
//! crash-recovery invariants are the oracle, checked by
//! sim/storage_sweep.zig's DST sweep.
//!
//! The read-fault dimension (`read_rot_pct`: latent sector errors surfacing
//! at read time on a store that opened clean) follows TigerBeetle's
//! storage-fault-injection lineage — its simulator models bit rot that a
//! read must detect, never silently serve.

const std = @import("std");
const storage = @import("zit").storage;

/// Per-fault trigger probabilities, each a percent in `0..=100` drawn
/// against fresh bits from the run PRNG. All zero (the default) is a
/// perfectly healthy disk — DST phase 2 "heals" by resetting to this.
pub const FaultCfg = struct {
    /// append() persists a strict prefix (`k < n`) and returns error.Io.
    short_write_pct: u8 = 0,
    /// sync() reports success without moving `durable_len` — the next
    /// crash then drops everything it claimed to have saved.
    lost_tail_sync_pct: u8 = 0,
    /// On crashRestart, force the cut exactly at `durable_len` (tear the
    /// whole unsynced tail) instead of a random point within it.
    crash_cut_at_durable_pct: u8 = 0,
    /// On crashRestart, additionally seed a few bytes before the cut as
    /// corrupt (payload-derived bitflip, never self-heals).
    crash_rot_pct: u8 = 0,
    /// Latent sector error: on a read of an INTACT byte, with this
    /// probability intern one offset within the read span as permanently
    /// corrupt (payload-derived mask, never self-heals) — bit rot surfacing
    /// at read time on a store that opened clean, independent of any crash.
    /// The distinct dimension from `crash_rot_pct`, which only fires at a
    /// crash boundary.
    read_rot_pct: u8 = 0,
    /// When the cut isn't forced to `durable_len`, sometimes round it down
    /// to a sector boundary (torn mid-sector writes are the common case on
    /// real disks).
    sector_round: u64 = 512,
    sector_round_pct: u8 = 0,
};

/// One `StorageFile` implementation over an in-memory byte buffer, standing
/// in for a real disk that can tear writes, lie about fsync, and flip bits.
pub const FaultyFile = struct {
    gpa: std.mem.Allocator,
    random: std.Random,
    bytes: std.ArrayList(u8),
    /// Watermark: bytes in `[0, durable_len)` survived the last true fsync.
    /// Monotonic non-decreasing for the life of the file — sync only moves
    /// it forward, and a crash cut is always chosen `>= durable_len`.
    durable_len: u64,
    /// Sorted, deduped offsets that read back flipped. Membership is
    /// permanent once set — the corresponding fault never self-heals.
    corrupt: std.ArrayList(u64),
    cfg: FaultCfg,

    /// Explicit upper bound on the simulated file (storage-track-plan.md
    /// §B): keeps the sweep's growth bounded and provable, not merely
    /// convenient.
    pub const max_bytes: u64 = 64 << 20;

    pub fn init(gpa: std.mem.Allocator, random: std.Random) FaultyFile {
        return .{
            .gpa = gpa,
            .random = random,
            .bytes = .empty,
            .durable_len = 0,
            .corrupt = .empty,
            .cfg = .{},
        };
    }

    pub fn deinit(self: *FaultyFile) void {
        self.bytes.deinit(self.gpa);
        self.corrupt.deinit(self.gpa);
        self.* = undefined;
    }

    pub fn storageFile(self: *FaultyFile) storage.StorageFile {
        return .{ .context = self, .vtable = &vtable };
    }

    /// Simulate power loss and reopen: pick a cut in `[durable_len, len]`
    /// (weighted toward exactly `durable_len`, sometimes sector-rounded),
    /// drop everything past it, and maybe rot a few bytes before it.
    pub fn crashRestart(self: *FaultyFile) void {
        std.debug.assert(self.durable_len <= self.bytes.items.len);
        const unsynced = self.bytes.items.len - self.durable_len;
        var cut: u64 = self.durable_len;
        if (unsynced > 0 and !drawPct(self.random, self.cfg.crash_cut_at_durable_pct)) {
            cut += self.random.uintAtMost(u64, unsynced);
            if (cut > self.durable_len and self.cfg.sector_round != 0 and
                drawPct(self.random, self.cfg.sector_round_pct))
            {
                const rounded = (cut / self.cfg.sector_round) * self.cfg.sector_round;
                cut = @max(rounded, self.durable_len);
            }
        }
        std.debug.assert(cut >= self.durable_len and cut <= self.bytes.items.len);

        if (cut > 0 and drawPct(self.random, self.cfg.crash_rot_pct)) {
            const budget = @min(cut, 4);
            const count = 1 + self.random.uintLessThan(u64, budget);
            var i: u64 = 0;
            while (i < count) : (i += 1) self.addCorrupt(self.random.uintLessThan(u64, cut));
        }

        self.bytes.shrinkRetainingCapacity(@intCast(cut));
        self.durable_len = cut;
        self.pruneCorruptAbove(cut);
        std.debug.assert(self.durable_len <= self.bytes.items.len);
        std.debug.assert(self.bytes.items.len <= max_bytes);
    }

    fn addCorrupt(self: *FaultyFile, off: u64) void {
        std.debug.assert(off < self.bytes.items.len);
        for (self.corrupt.items) |existing| {
            if (existing == off) return;
        }
        // Best-effort: missing one injected fault under real allocation
        // pressure only means slightly less fault coverage this round,
        // never a correctness gap in the store under test.
        self.corrupt.append(self.gpa, off) catch return;
        std.mem.sort(u64, self.corrupt.items, {}, std.sort.asc(u64));
    }

    fn pruneCorruptAbove(self: *FaultyFile, cut: u64) void {
        var write: usize = 0;
        for (self.corrupt.items) |off| {
            if (off < cut) {
                self.corrupt.items[write] = off;
                write += 1;
            }
        }
        self.corrupt.shrinkRetainingCapacity(write);
    }

    const vtable: storage.StorageFile.VTable = .{
        .append = appendImpl,
        .pread = preadImpl,
        .sync = syncImpl,
        .size = sizeImpl,
        .truncate = truncateImpl,
        .lock = lockImpl,
    };

    /// One writer by construction, so there is no other writer to exclude and
    /// no other writer's appends to re-read the length for.
    fn lockImpl(_: *anyopaque) storage.StorageError!void {}

    fn appendImpl(ctx: *anyopaque, data: []const u8) storage.StorageError!void {
        const self: *FaultyFile = @ptrCast(@alignCast(ctx));
        std.debug.assert(self.bytes.items.len + data.len <= max_bytes);
        if (data.len > 0 and drawPct(self.random, self.cfg.short_write_pct)) {
            // k < n by construction: a "short write" that persisted every
            // requested byte would not be short.
            const k = self.random.uintLessThan(usize, data.len);
            self.bytes.appendSlice(self.gpa, data[0..k]) catch return error.Io;
            return error.Io;
        }
        self.bytes.appendSlice(self.gpa, data) catch return error.Io;
    }

    fn preadImpl(ctx: *anyopaque, offset: u64, buf: []u8) storage.StorageError!usize {
        const self: *FaultyFile = @ptrCast(@alignCast(ctx));
        // StorageFile contract: an offset at or past EOF reads 0 bytes — it
        // is neither an error nor a precondition violation, so the sim must
        // not assert a stricter bound than the capability it stands in for.
        if (offset >= self.bytes.items.len) return 0;
        const avail = self.bytes.items.len - offset;
        const n = @min(avail, buf.len);
        if (n == 0) return 0;
        const start: usize = @intCast(offset);
        @memcpy(buf[0..n], self.bytes.items[start..][0..n]);
        // Latent sector error: intern one offset in this read span as
        // permanently corrupt, so it flips here AND on every later read of
        // it (the `crash_rot`/`corruptInPlace` discipline: payload-derived,
        // never self-healing). Drawn only when the knob is armed — a healthy
        // disk (`read_rot_pct == 0`) draws nothing, so an existing sweep's
        // PRNG stream is byte-identical.
        if (drawPct(self.random, self.cfg.read_rot_pct)) {
            self.addCorrupt(offset + self.random.uintLessThan(u64, n));
        }
        for (self.corrupt.items) |off| {
            if (off >= offset and off < offset + n) {
                buf[@intCast(off - offset)] ^= corruptionMask(off);
            }
        }
        return n;
    }

    fn syncImpl(ctx: *anyopaque) storage.StorageError!void {
        const self: *FaultyFile = @ptrCast(@alignCast(ctx));
        if (drawPct(self.random, self.cfg.lost_tail_sync_pct)) return; // lies: durable_len unmoved
        self.durable_len = self.bytes.items.len;
    }

    fn sizeImpl(ctx: *anyopaque) storage.StorageError!u64 {
        const self: *FaultyFile = @ptrCast(@alignCast(ctx));
        return self.bytes.items.len;
    }

    fn truncateImpl(ctx: *anyopaque, len: u64) storage.StorageError!void {
        const self: *FaultyFile = @ptrCast(@alignCast(ctx));
        std.debug.assert(len <= max_bytes);
        const len_usize: usize = @intCast(len);
        if (len_usize <= self.bytes.items.len) {
            self.bytes.shrinkRetainingCapacity(len_usize);
        } else {
            const old = self.bytes.items.len;
            self.bytes.resize(self.gpa, len_usize) catch return error.Io;
            @memset(self.bytes.items[old..], 0);
        }
        self.pruneCorruptAbove(len);
        self.durable_len = @min(self.durable_len, len);
    }
};

fn drawPct(random: std.Random, pct: u8) bool {
    std.debug.assert(pct <= 100);
    if (pct == 0) return false;
    if (pct == 100) return true;
    return random.uintLessThan(u8, 100) < pct;
}

/// Payload-derived, pure function of the offset: every retry corrupts
/// identically (storage-track-plan.md §B). Always nonzero, so it always
/// actually flips something.
fn corruptionMask(off: u64) u8 {
    const h = std.hash.Wyhash.hash(0, std.mem.asBytes(&off));
    return @truncate(1 | (h & 7));
}

test "faulty_file §B: pread at or past EOF reads 0 bytes (StorageFile contract)" {
    var prng = std.Random.DefaultPrng.init(6);
    var faulty = FaultyFile.init(std.testing.allocator, prng.random());
    defer faulty.deinit();

    const file = faulty.storageFile();
    try file.append("ABCD");
    try file.sync();

    var buf: [4]u8 = undefined;
    try std.testing.expectEqual(@as(usize, 0), try file.pread(4, &buf)); // exactly at EOF
    try std.testing.expectEqual(@as(usize, 0), try file.pread(99, &buf)); // well past EOF
}

test "faulty_file §B: short write persists a k<n prefix and returns error.Io" {
    var prng = std.Random.DefaultPrng.init(1);
    var faulty = FaultyFile.init(std.testing.allocator, prng.random());
    defer faulty.deinit();
    faulty.cfg.short_write_pct = 100;

    const file = faulty.storageFile();
    try std.testing.expectError(error.Io, file.append("12345678"));
    const size = try file.size();
    try std.testing.expect(size < 8);

    var buf: [8]u8 = undefined;
    const n = try file.pread(0, buf[0..@intCast(size)]);
    try std.testing.expectEqual(size, n);
    try std.testing.expectEqualStrings("12345678"[0..@intCast(size)], buf[0..n]);
}

test "faulty_file §B: crash-restart tears the unsynced tail within [durable_len, size]" {
    var prng = std.Random.DefaultPrng.init(2);
    var faulty = FaultyFile.init(std.testing.allocator, prng.random());
    defer faulty.deinit();
    faulty.cfg.crash_rot_pct = 0; // isolate the tear from corruption

    const file = faulty.storageFile();
    try file.append("AAAA");
    try file.sync();
    try file.append("BBBB");
    faulty.crashRestart();

    const size = try file.size();
    try std.testing.expect(size >= 4 and size <= 8);
    var buf: [4]u8 = undefined;
    _ = try file.pread(0, &buf);
    try std.testing.expectEqualStrings("AAAA", &buf);
}

test "faulty_file §B: fsync-lost-tail drops every unsynced byte on the next crash" {
    var prng = std.Random.DefaultPrng.init(3);
    var faulty = FaultyFile.init(std.testing.allocator, prng.random());
    defer faulty.deinit();

    const file = faulty.storageFile();
    try file.append("AAAA");
    try file.sync();
    try file.append("BBBB");
    faulty.cfg.lost_tail_sync_pct = 100;
    try file.sync(); // lies: claims success, durable_len stays at 4
    faulty.cfg.crash_cut_at_durable_pct = 100;
    faulty.crashRestart();

    try std.testing.expectEqual(@as(u64, 4), try file.size());
}

test "faulty_file §B: a bitflip is payload-derived and never self-heals on retry" {
    var prng = std.Random.DefaultPrng.init(4);
    var faulty = FaultyFile.init(std.testing.allocator, prng.random());
    defer faulty.deinit();

    const file = faulty.storageFile();
    try file.append("AAAAAAAA");
    try file.sync();
    faulty.cfg.crash_cut_at_durable_pct = 100; // cut lands at len 8, nothing torn
    faulty.cfg.crash_rot_pct = 100; // force at least one corrupt offset
    faulty.crashRestart();
    try std.testing.expect(faulty.corrupt.items.len > 0);
    const off = faulty.corrupt.items[0];

    var first: [1]u8 = undefined;
    var second: [1]u8 = undefined;
    _ = try file.pread(off, &first);
    _ = try file.pread(off, &second);
    try std.testing.expectEqual(first[0], second[0]);
    try std.testing.expect(first[0] != 'A');
}

test "faulty_file §B: a read fault returns altered bytes without signalling an error" {
    var prng = std.Random.DefaultPrng.init(5);
    var faulty = FaultyFile.init(std.testing.allocator, prng.random());
    defer faulty.deinit();

    const file = faulty.storageFile();
    try file.append("ZZZZZZZZ");
    try file.sync();
    faulty.cfg.crash_cut_at_durable_pct = 100;
    faulty.cfg.crash_rot_pct = 100;
    faulty.crashRestart();
    try std.testing.expect(faulty.corrupt.items.len > 0);
    const off = faulty.corrupt.items[0];

    var buf: [1]u8 = undefined;
    const n = try file.pread(off, &buf); // no error -- a read fault, not a device fault
    try std.testing.expectEqual(@as(usize, 1), n);
    try std.testing.expect(buf[0] != 'Z');
}

test "faulty_file §B: read_rot flips an intact byte at read time and never self-heals" {
    var prng = std.Random.DefaultPrng.init(7);
    var faulty = FaultyFile.init(std.testing.allocator, prng.random());
    defer faulty.deinit();

    const file = faulty.storageFile();
    try file.append("ABCDEFGH");
    try file.sync(); // fully durable, no crash -- the store "opened clean"
    faulty.cfg.read_rot_pct = 100; // latent sector error engages on the next read
    std.debug.assert(faulty.corrupt.items.len == 0);

    var buf: [8]u8 = undefined;
    const n = try file.pread(0, &buf); // no error surfaced -- rot is served, not signalled
    try std.testing.expectEqual(@as(usize, 8), n);
    try std.testing.expect(!std.mem.eql(u8, "ABCDEFGH", &buf)); // at least one byte flipped
    try std.testing.expect(faulty.corrupt.items.len > 0); // interned permanently
    const off = faulty.corrupt.items[0];

    // The same offset reads back identically corrupt with the knob disarmed:
    // payload-derived and permanent, never self-healing on a later read.
    faulty.cfg.read_rot_pct = 0;
    var a: [1]u8 = undefined;
    var b: [1]u8 = undefined;
    _ = try file.pread(off, &a);
    _ = try file.pread(off, &b);
    try std.testing.expectEqual(a[0], b[0]);
    try std.testing.expect(a[0] != "ABCDEFGH"[@intCast(off)]);
}