//! Deterministic working-tree crash simulator (H2, native harness design — no
//! RFC clause touches the working tree). A decorator `std.Io` that sits in front
//! of a healthy backend io and, at a single seeded operation index, "crashes":
//! it kills the run's forward progress the way a power loss does — the crashing
//! mutation applies at most a torn prefix, and every mutating op after it is a
//! no-op error, so whatever the checkout layer's own rollback tries to do next
//! moves no byte. The seed is the whole universe: the same seed reproduces the
//! same cut and the same on-disk aftermath exactly.
//!
//! The crash-point enumeration technique — walk the operation sequence and cut
//! at every reachable boundary, then assert recovery from each cut — is
//! TigerBeetle's storage-fault-injection lineage (the VOPR's disk model),
//! adapted here from block storage to the POSIX directory surface `update.zig`
//! and `checkout.zig` write through. It mirrors sim/faulty_file.zig's
//! discipline: every fault is a draw from the run PRNG, corruption is a pure
//! function of the payload so a retry never self-heals, and the healthy default
//! (no `crash_at_op`) is a perfect disk that DST phase 2 recovers over.
//!
//! Mechanism: the decorator keeps the backend's `userdata` and copies the
//! backend `VTable`, overriding only the mutating entries. The non-mutating
//! entries stay the backend's own function pointers, so reads/opens/stats never
//! lie — a crash tears WRITES, never the ability to observe what survived.
//! The active decorator is a single-run install (one sweep at a time, single
//! threaded, exactly like update.zig's `test_fault`); the overrides reach it
//! through `active`.
const std = @import("std");
const Io = std.Io;
const Dir = std.Io.Dir;
const File = std.Io.File;
/// The one decorator wired for the current run. Set by `install`, cleared by
/// `uninstall`; the vtable overrides read it. Single-threaded by construction
/// (the DST sweep drives one operation at a time), the same shape update.zig's
/// `test_fault` global uses to reach a file-private fault seam.
var active: ?*FaultyDir = null;
/// Crash schedule for one run. `crash_at_op == null` is a perfect disk (the
/// healthy default DST phase 2 recovers over); a set value crashes as the
/// `crash_at_op`-th mutating operation begins.
pub const CrashCfg = struct {
/// One-indexed mutating-operation number to crash at, or null for never.
/// Counting covers every write/sync/create/delete/rename the run performs,
/// so a value in `[1, total_ops]` selects a reachable cut.
crash_at_op: ?u64 = null,
/// When true, the crashing WRITE first persists a random strict prefix of
/// its bytes (a torn write) before failing — the byte-granular power-loss
/// tear that leaves a file matching neither its pre- nor post-image. When
/// false the crash is operation-granular (the write does not apply at all).
torn_write: bool = false,
/// When true, the torn prefix is additionally bit-flipped with a
/// payload-derived mask (pure function of the offset, never self-heals),
/// mirroring faulty_file.zig's `corruptionMask`.
corrupt_torn: bool = false,
/// When true, model a POWER LOSS rather than a mere I/O fault: bytes written
/// but never `fsync`ed do not survive. Each open file carries a durable
/// watermark (see `Tracked`); at `close` the file is truncated back to it.
///
/// WITHOUT THIS THE DECORATOR CANNOT SEE AN fsync AT ALL. `wrapSync` only
/// gates — a proceeding sync delegates and changes nothing observable, and a
/// crashing one errors — so every byte a run wrote reaches the real
/// filesystem whether or not the run ever synced. That makes "fsync the new
/// generation before renaming it over `.zit`" an assertion rather than a
/// tested property: delete the fsync and every crash cut still finds a
/// complete file. `sim/faulty_file.zig` carries `durable_len` for exactly
/// this reason; this is its directory-surface counterpart. Off by default so
/// the existing checkout sweep's fault model is unchanged.
lose_unsynced: bool = false,
/// Model the other half of a power loss: a create, rename or delete is not
/// durable until its DIRECTORY is fsynced. `lose_unsynced` covers a file's
/// bytes; this covers its name. Each such operation is recorded until an
/// fsync of its directory makes it durable, and `dropUnsyncedDirents`
/// reverts the rest, newest first, as a power cut would: a rename puts
/// back whatever its destination held, a create disappears, a delete
/// comes back.
///
/// Without it, a rename is durable the moment it returns, and "fsync the
/// directory after renaming the new generation over `.zit`" is an
/// assertion no sweep can test. Needs `gpa`, for the bytes a rename or a
/// delete replaced. Off by default so every existing sweep's model is
/// unchanged.
lose_unsynced_dirents: bool = false,
gpa: ?std.mem.Allocator = null,
/// Fire `hook` immediately AFTER the mutating operation with this index
/// completes. Zero-cost when null.
///
/// AFTER, and that is the whole point. The interesting window in a rewrite
/// is between its last durable write and its rename — `rewriteAndSwap`
/// re-stats the source and re-validates its tail in there, with no mutating
/// operation of its own to hang a before-hook on. A sweep that knows how
/// many operations follow that window names it by counting back from the
/// total. The hook is how a sweep plays a concurrent writer at a
/// deterministic point instead of racing a thread.
after_op: ?u64 = null,
/// The callback `after_op` fires, and its context. It MUST drive the healthy
/// backend io (never the decorated one), or it re-enters the gate and shifts
/// every later operation index.
hook: ?*const fn (ctx: ?*anyopaque) void = null,
hook_ctx: ?*anyopaque = null,
};
/// One open file's write watermarks, for the `lose_unsynced` power-loss model.
/// `dirty` is the highest byte offset any write has reached; `durable` is what
/// `dirty` was at the last successful `fsync`. Keyed by OS handle, which is
/// unique among simultaneously-open descriptors; the entry is dropped at close,
/// so a recycled handle never inherits a stale watermark.
///
/// THAT LAST SENTENCE IS AN INVARIANT WITH A TEETH-BEARING CHECK, not a hope.
/// It holds only while every open that writes and every close of the same file
/// go through THIS decorator. A file written through the decorated io but closed
/// through the healthy backend would leave a live entry behind, and the OS would
/// then hand the same `File.Handle` to an unrelated later open — whose close
/// would truncate an innocent file to a foreign watermark, which the sweep's
/// oracle would report as a hybrid store for a reason having nothing to do with
/// the property. `loseUnsynced` therefore compares the tracked `dirty` against
/// the file's real size before it truncates anything, and treats `size > dirty`
/// as a harness bug (panic) rather than a fault to inject: that inequality is
/// exactly the signature of both this stale-entry case and of bytes having
/// reached the file through a vtable entry the decorator does not override.
///
/// That comparison covers the PARTIALLY tracked file. The wholly untracked one —
/// a closing file with bytes and no slot at all, which is what a write reaching
/// the disk through an un-overridden entry actually produces — never reaches it,
/// because the lookup loop skips the file entirely. `loseUnsynced` carries a
/// second, complementary panic for that case; see there.
const Tracked = struct {
handle: File.Handle,
dirty: u64,
durable: u64,
};
/// Bound on simultaneously-tracked open files. A rewrite holds two (`.zit` and
/// the temp) and the sweeps' workspaces a handful; 32 is a tripwire, not a
/// budget, and overflowing it is a harness bug rather than a fault to inject.
const max_tracked = 32;
/// Bound on directory operations awaiting a directory fsync, and on the length
/// of the names they carry. Tripwires, like `max_tracked`: a sweep workspace
/// makes a handful of each, and overflowing them is a harness bug.
const max_dirents = 32;
const max_dirent_name = 128;
/// One create, rename or delete not yet made durable by an fsync of its
/// directory. Holds its own handle on the directory, so it can be reverted
/// after the run has closed its handles.
const Dirent = struct {
kind: enum { create, rename, delete },
dir: Dir,
dir_inode: File.INode,
/// create, delete: the name. rename: the destination.
name_buf: [max_dirent_name]u8,
name_len: usize,
/// rename: the source.
from_buf: [max_dirent_name]u8,
from_len: usize,
/// rename: what the destination held before, if it existed. delete: the
/// deleted file's bytes.
prev: ?[]u8,
fn name(self: *const Dirent) []const u8 {
return self.name_buf[0..self.name_len];
}
fn from(self: *const Dirent) []const u8 {
return self.from_buf[0..self.from_len];
}
};
pub const FaultyDir = struct {
/// The healthy io every non-crashing op delegates to.
backend: Io,
/// The backend's own vtable, kept so the overrides can call through it.
backend_vtable: *const Io.VTable,
/// A copy of `backend_vtable` with the mutating entries pointed at the
/// overrides below. `io()` hands out `&self.vtable`, so `self` must outlive
/// every use (the caller pins it as a `var`).
vtable: Io.VTable,
random: std.Random,
cfg: CrashCfg,
/// Mutating operations begun so far (monotonic; the crash index is compared
/// against it).
op_index: u64 = 0,
/// Latched once the cut fires; every later mutating op is a frozen no-op.
crashed: bool = false,
/// The operation index the crash landed at (0 until it fires) — the value a
/// failure report prints so the cut is auditable.
crash_op_index: u64 = 0,
/// Per-open-file write watermarks; live only while `cfg.lose_unsynced`.
tracked: [max_tracked]Tracked = undefined,
tracked_n: usize = 0,
/// Set the first time a write or a `setLength` moves a watermark — i.e. the
/// first time this decorator saw a byte destined for a file. Read only by
/// `loseUnsynced`, to tell "this run never wrote anything, so an untracked
/// close is unremarkable" from "this run writes, and yet here is a file with
/// bytes it never saw" (a harness bug; see there).
wrote_any: bool = false,
/// How many files were closed while holding a tracked slot with `dirty > 0`
/// — i.e. how many times the power-loss model actually had a file whose
/// unsynced tail it could have discarded.
///
/// EXPOSED BECAUSE A SWEEP CANNOT OTHERWISE TELL THE MODEL RAN. With
/// `lose_unsynced` set and a healthy schedule, `dirty == durable` at close
/// and nothing is truncated — correct, and indistinguishable from the model
/// being switched off entirely. A run that exists to test "fsync before
/// rename" degenerates, silently and greenly, into "a healthy rewrite
/// produces the post image" the moment the model stops engaging, so the run
/// must be able to assert that it engaged. `crashed`/`crash_op_index` are
/// the same affordance for the crash schedule.
dirty_closes: u64 = 0,
/// Directory operations not yet made durable; live only while
/// `cfg.lose_unsynced_dirents`.
dirents: [max_dirents]Dirent = undefined,
dirents_n: usize = 0,
/// Directory operations recorded over the whole run, durable or not: how a
/// sweep asserts the model engaged.
dirents_seen: u64 = 0,
/// The injected failure. `AccessDenied` is a member of every mutating error
/// set on this path (write/sync/create/delete/rename), so one constant maps
/// cleanly into all of them; update.zig collapses any of them to its own
/// `error.Storage`, exactly as a real device fault would.
const injected = error.AccessDenied;
pub fn init(backend: Io, random: std.Random, cfg: CrashCfg) FaultyDir {
var v = backend.vtable.*;
v.fileWritePositional = wrapWrite;
v.fileSync = wrapSync;
v.dirCreateFile = wrapCreateFile;
v.dirCreateFileAtomic = wrapCreateFileAtomic;
v.dirCreateDir = wrapCreateDir;
v.dirDeleteFile = wrapDeleteFile;
v.dirDeleteDir = wrapDeleteDir;
v.dirRename = wrapRename;
// None of these three GATES — they are not counted as mutating
// operations, so wiring them leaves every existing sweep's op numbering
// (and therefore its cuts) byte-for-byte what it was. They exist only to
// maintain the `lose_unsynced` watermarks: `setLength` moves both marks,
// `close` is where an unsynced tail is discarded, and a read-only `open`
// is registered so `loseUnsynced` can tell "this handle wrote nothing"
// from "bytes reached this file behind the decorator's back".
v.fileSetLength = wrapSetLength;
v.fileClose = wrapClose;
v.dirOpenFile = wrapOpenFile;
return .{
.backend = backend,
.backend_vtable = backend.vtable,
.vtable = v,
.random = random,
.cfg = cfg,
};
}
/// The decorated io. Shares the backend's `userdata`, so the un-overridden
/// backend entries receive exactly the context they expect.
pub fn io(self: *FaultyDir) Io {
return .{ .userdata = self.backend.userdata, .vtable = &self.vtable };
}
pub fn install(self: *FaultyDir) void {
std.debug.assert(active == null); // one decorator at a time
active = self;
}
pub fn uninstall() void {
active = null;
}
/// The power cut for directory entries: revert, newest first, every
/// create, rename and delete no directory fsync made durable. Goes through
/// the backend directly, so it counts no operation and is never frozen.
pub fn dropUnsyncedDirents(self: *FaultyDir) void {
const gpa = self.cfg.gpa orelse return;
const io_b = self.backend;
while (self.dirents_n > 0) {
self.dirents_n -= 1;
var d = &self.dirents[self.dirents_n];
switch (d.kind) {
.create => d.dir.deleteFile(io_b, d.name()) catch {},
.rename => {
d.dir.rename(d.name(), d.dir, d.from(), io_b) catch {};
if (d.prev) |bytes| d.dir.writeFile(io_b, .{ .sub_path = d.name(), .data = bytes }) catch {};
},
.delete => if (d.prev) |bytes| d.dir.writeFile(io_b, .{ .sub_path = d.name(), .data = bytes }) catch {},
}
if (d.prev) |bytes| gpa.free(bytes);
d.dir.close(io_b);
}
}
/// Forget the directory operations recorded so far without reverting
/// them, releasing what they hold. For a run that ends without a cut.
pub fn keepDirents(self: *FaultyDir) void {
const gpa = self.cfg.gpa orelse return;
while (self.dirents_n > 0) {
self.dirents_n -= 1;
const d = &self.dirents[self.dirents_n];
if (d.prev) |bytes| gpa.free(bytes);
d.dir.close(self.backend);
}
}
/// Record a directory operation about to happen in `dir`. `prev_name`, when
/// it exists, is read first: its bytes are what a revert puts back.
fn noteDirent(self: *FaultyDir, kind: @FieldType(Dirent, "kind"), dir: Dir, name: []const u8, from: []const u8, prev_name: ?[]const u8) void {
if (!self.cfg.lose_unsynced_dirents) return;
const gpa = self.cfg.gpa orelse @panic("faulty_dir: lose_unsynced_dirents needs gpa (harness bug)");
if (self.dirents_n >= max_dirents) @panic("faulty_dir: dirent table exhausted (harness bug)");
if (name.len > max_dirent_name or from.len > max_dirent_name) @panic("faulty_dir: dirent name too long (harness bug)");
const own = dir.openDir(self.backend, ".", .{ .iterate = true }) catch @panic("faulty_dir: cannot hold the directory (harness bug)");
const st = own.stat(self.backend) catch @panic("faulty_dir: cannot stat the directory (harness bug)");
var d: Dirent = .{
.kind = kind,
.dir = own,
.dir_inode = st.inode,
.name_buf = undefined,
.name_len = name.len,
.from_buf = undefined,
.from_len = from.len,
.prev = null,
};
@memcpy(d.name_buf[0..name.len], name);
@memcpy(d.from_buf[0..from.len], from);
if (prev_name) |pn| {
d.prev = dir.readFileAlloc(self.backend, pn, gpa, .limited(64 << 20)) catch null;
}
self.dirents[self.dirents_n] = d;
self.dirents_n += 1;
self.dirents_seen += 1;
}
/// An fsync of a directory makes every operation recorded in it durable.
fn noteDirSync(self: *FaultyDir, file: File) void {
if (!self.cfg.lose_unsynced_dirents or self.dirents_n == 0) return;
const st = file.stat(self.backend) catch return;
if (st.kind != .directory) return;
const gpa = self.cfg.gpa.?;
var kept: usize = 0;
for (self.dirents[0..self.dirents_n]) |d| {
if (d.dir_inode == st.inode) {
if (d.prev) |bytes| gpa.free(bytes);
d.dir.close(self.backend);
} else {
self.dirents[kept] = d;
kept += 1;
}
}
self.dirents_n = kept;
}
fn exists(self: *FaultyDir, dir: Dir, name: []const u8) bool {
dir.access(self.backend, name, .{}) catch return false;
return true;
}
const Decision = enum { proceed, crash_now, frozen };
fn gate(self: *FaultyDir) Decision {
if (self.crashed) return .frozen;
self.op_index += 1;
if (self.cfg.crash_at_op) |k| {
if (self.op_index >= k) {
self.crashed = true;
self.crash_op_index = self.op_index;
return .crash_now;
}
}
return .proceed;
}
/// Fire the `after_op` callback if this op is the one it names. Called on
/// the PROCEED path only: an operation that failed did not complete, so the
/// window the hook models never opened.
fn fireHook(self: *FaultyDir) void {
const n = self.cfg.after_op orelse return;
if (self.op_index != n) return;
const h = self.cfg.hook orelse return;
h(self.cfg.hook_ctx);
}
/// The watermark slot for `handle`, created at `{0, 0}` on first sight.
/// EXHAUSTION PANICS RATHER THAN DEGRADING. Returning null here and letting
/// the callers `orelse return` would make `noteWrite`/`noteSync` silent
/// no-ops for that file, which switches the `lose_unsynced` power-loss model
/// off for it — the oracle would then stop firing instead of failing. An
/// assert alone is not enough: `std.debug.assert` compiles to nothing in
/// `ReleaseFast`/`ReleaseSmall`, which is precisely where a silently
/// disabled oracle would go unnoticed.
fn slot(self: *FaultyDir, handle: File.Handle) *Tracked {
for (self.tracked[0..self.tracked_n]) |*t| {
if (t.handle == handle) return t;
}
if (self.tracked_n >= max_tracked) @panic("faulty_dir: tracked-file table exhausted (harness bug)");
self.tracked[self.tracked_n] = .{ .handle = handle, .dirty = 0, .durable = 0 };
self.tracked_n += 1;
return &self.tracked[self.tracked_n - 1];
}
fn noteWrite(self: *FaultyDir, file: File, end: u64) void {
if (!self.cfg.lose_unsynced) return;
const t = self.slot(file.handle);
if (end > t.dirty) t.dirty = end;
self.wrote_any = true;
}
fn noteSync(self: *FaultyDir, file: File) void {
if (!self.cfg.lose_unsynced) return;
const t = self.slot(file.handle);
t.durable = t.dirty;
}
/// `setLength` makes the file exactly `len` bytes and is itself a metadata
/// operation that does not imply a data flush, so the dirty mark moves to
/// `len` and the durable mark can only shrink to it.
fn noteSetLength(self: *FaultyDir, file: File, len: u64) void {
if (!self.cfg.lose_unsynced) return;
const t = self.slot(file.handle);
t.dirty = len;
if (t.durable > len) t.durable = len;
self.wrote_any = true;
}
/// Power loss, applied at the moment the descriptor goes away: everything
/// written past the last successful `fsync` is discarded. Truncating through
/// the still-open handle is what makes this observable to a later reader —
/// and it is correct even after a rename, since the handle names the inode,
/// not the path.
fn loseUnsynced(self: *FaultyDir, ud: ?*anyopaque, files: []const File) void {
if (!self.cfg.lose_unsynced) return;
for (files) |file| {
var i: usize = 0;
var found = false;
while (i < self.tracked_n) : (i += 1) {
if (self.tracked[i].handle != file.handle) continue;
found = true;
const t = self.tracked[i];
// THE TRACKING MUST ACCOUNT FOR EVERY BYTE IN THE FILE, or the
// truncation below is not a power loss but a corruption the
// harness invented. `size > dirty` means either bytes arrived
// through a vtable entry this decorator does not override, or
// this entry is stale and the OS recycled its handle onto an
// unrelated file. Both are harness bugs, and both would show up
// downstream as an invariant violation attributed to the code
// under test — so they stop here, loudly, in every build mode.
if (self.backend_vtable.fileStat(ud, file)) |st| {
if (st.size > t.dirty) @panic("faulty_dir: untracked bytes in a tracked file (harness bug)");
} else |_| {}
if (t.dirty > 0) self.dirty_closes += 1;
if (t.dirty > t.durable) {
self.backend_vtable.fileSetLength(ud, file, t.durable) catch {};
}
self.tracked[i] = self.tracked[self.tracked_n - 1];
self.tracked_n -= 1;
break;
}
// THE COMPLEMENTARY HALF of the `size > dirty` guard above, and the
// half that catches the regression that guard is written for. A file
// whose bytes arrived through a vtable entry this decorator does not
// override has NO slot at all, so the loop above skips it entirely
// and never reaches its panic: the loud case is the PARTIALLY tracked
// one, and the silent case is the untracked one. Here a closing file
// with no slot but a non-empty extent, in a run that demonstrably
// does write through this decorator, is exactly the same harness bug
// — untracked bytes — and stops the same way, in every build mode.
//
// Narrow on purpose: a run that never wrote (`wrote_any` false) or a
// zero-length file has no untracked bytes to account for, so a
// read-only open closed through the decorated io is not a bug.
if (!found and self.wrote_any) {
if (self.backend_vtable.fileStat(ud, file)) |st| {
if (st.size > 0) @panic("faulty_dir: untracked file with bytes at close (harness bug)");
} else |_| {}
}
}
}
/// FROZEN, BUT NOT COUNTED — the two are different properties and only the
/// first was argued where these entries are wired. Not counted, so every
/// existing sweep's op numbering is byte-for-byte what it was. Frozen
/// anyway, because a crashed process cannot truncate a file: delegating here
/// after the cut has latched would let a length change reach the real
/// filesystem during the crashed run's unwind, which no power loss permits.
/// `AccessDenied` is a member of `SetLengthError`, so the freeze reports
/// itself exactly as every other frozen mutation does.
///
/// `loseUnsynced` deliberately calls the backend entry directly rather than
/// coming through here: that truncation is the power loss itself, not an
/// operation the crashed process performs.
fn wrapSetLength(ud: ?*anyopaque, file: File, len: u64) File.SetLengthError!void {
const self = active.?;
if (self.crashed) return injected;
try self.backend_vtable.fileSetLength(ud, file, len);
self.noteSetLength(file, len);
}
fn wrapClose(ud: ?*anyopaque, files: []const File) void {
const self = active.?;
self.loseUnsynced(ud, files);
self.backend_vtable.fileClose(ud, files);
}
fn wrapWrite(
ud: ?*anyopaque,
file: File,
header: []const u8,
data: []const []const u8,
splat: usize,
offset: u64,
) File.WritePositionalError!usize {
const self = active.?;
switch (self.gate()) {
.proceed => {
const n = try self.backend_vtable.fileWritePositional(ud, file, header, data, splat, offset);
self.noteWrite(file, offset + n);
self.fireHook();
return n;
},
.frozen => return injected,
.crash_now => {
// A torn prefix is by construction NOT durable — it is the page
// cache surviving into the file, not an fsync — so the dirty
// mark moves and the durable mark does not.
if (self.cfg.torn_write) self.applyTorn(ud, file, data, offset);
return injected;
},
}
}
/// Persist a strict prefix of the crashing write (the torn tail a power loss
/// leaves in the page cache), optionally payload-corrupted. Best-effort: a
/// backend hiccup here only means a slightly shorter tear, never a false
/// invariant verdict.
fn applyTorn(self: *FaultyDir, ud: ?*anyopaque, file: File, data: []const []const u8, offset: u64) void {
for (data) |buf| {
if (buf.len < 2) continue;
const k = 1 + self.random.uintLessThan(usize, buf.len - 1); // 1..=len-1
if (self.cfg.corrupt_torn) {
var scratch: [512]u8 = undefined;
const n = @min(k, scratch.len);
@memcpy(scratch[0..n], buf[0..n]);
var i: usize = 0;
while (i < n) : (i += 1) scratch[i] ^= corruptionMask(offset + i);
const w = self.backend_vtable.fileWritePositional(ud, file, &.{}, &.{scratch[0..n]}, 1, offset) catch 0;
self.noteWrite(file, offset + w);
} else {
const w = self.backend_vtable.fileWritePositional(ud, file, &.{}, &.{buf[0..k]}, 1, offset) catch 0;
self.noteWrite(file, offset + w);
}
return;
}
}
fn wrapSync(ud: ?*anyopaque, file: File) File.SyncError!void {
const self = active.?;
switch (self.gate()) {
.proceed => {
try self.backend_vtable.fileSync(ud, file);
self.noteSync(file);
self.noteDirSync(file);
self.fireHook();
return;
},
.frozen, .crash_now => return injected,
}
}
/// A read-only open is not a mutation: it is not gated, not counted, and
/// cannot fail here. It IS registered, because `loseUnsynced`'s untracked-file
/// panic exists to catch bytes that reached a file through a vtable entry this
/// decorator does not override, and a handle that was only ever read is the
/// one legitimate way to close a non-empty file with nothing to account for.
/// Seeding both watermarks at the file's current length says exactly that —
/// every byte present is already durable — so the close path finds the slot,
/// truncates nothing, and drops it.
fn wrapOpenFile(ud: ?*anyopaque, dir: Dir, sub_path: []const u8, opts: Dir.OpenFileOptions) File.OpenError!File {
const self = active.?;
const f = try self.backend_vtable.dirOpenFile(ud, dir, sub_path, opts);
if (self.cfg.lose_unsynced) {
const size: u64 = if (self.backend_vtable.fileStat(ud, f)) |st| st.size else |_| 0;
const t = self.slot(f.handle);
t.dirty = size;
t.durable = size;
}
return f;
}
fn wrapCreateFile(ud: ?*anyopaque, dir: Dir, sub_path: []const u8, opts: Dir.CreateFileOptions) File.OpenError!File {
const self = active.?;
switch (self.gate()) {
.proceed => {
const new = self.cfg.lose_unsynced_dirents and !self.exists(dir, sub_path);
const f = try self.backend_vtable.dirCreateFile(ud, dir, sub_path, opts);
if (new) self.noteDirent(.create, dir, sub_path, "", null);
self.fireHook();
return f;
},
.frozen, .crash_now => return injected,
}
}
fn wrapCreateFileAtomic(ud: ?*anyopaque, dir: Dir, sub_path: []const u8, opts: Dir.CreateFileAtomicOptions) Dir.CreateFileAtomicError!File.Atomic {
const self = active.?;
switch (self.gate()) {
.proceed => {
const f = try self.backend_vtable.dirCreateFileAtomic(ud, dir, sub_path, opts);
self.fireHook();
return f;
},
.frozen, .crash_now => return injected,
}
}
fn wrapCreateDir(ud: ?*anyopaque, dir: Dir, sub_path: []const u8, perm: Dir.Permissions) Dir.CreateDirError!void {
const self = active.?;
switch (self.gate()) {
.proceed => {
try self.backend_vtable.dirCreateDir(ud, dir, sub_path, perm);
self.fireHook();
return;
},
.frozen, .crash_now => return injected,
}
}
fn wrapDeleteFile(ud: ?*anyopaque, dir: Dir, sub_path: []const u8) Dir.DeleteFileError!void {
const self = active.?;
switch (self.gate()) {
.proceed => {
if (self.cfg.lose_unsynced_dirents and self.exists(dir, sub_path))
self.noteDirent(.delete, dir, sub_path, "", sub_path);
try self.backend_vtable.dirDeleteFile(ud, dir, sub_path);
self.fireHook();
return;
},
.frozen, .crash_now => return injected,
}
}
fn wrapDeleteDir(ud: ?*anyopaque, dir: Dir, sub_path: []const u8) Dir.DeleteDirError!void {
const self = active.?;
switch (self.gate()) {
.proceed => {
try self.backend_vtable.dirDeleteDir(ud, dir, sub_path);
self.fireHook();
return;
},
.frozen, .crash_now => return injected,
}
}
fn wrapRename(ud: ?*anyopaque, old_dir: Dir, old_sub_path: []const u8, new_dir: Dir, new_sub_path: []const u8) Dir.RenameError!void {
const self = active.?;
switch (self.gate()) {
.proceed => {
// One directory is all a sweep renames within; the revert moves
// the destination back to the source in `new_dir`.
if (self.cfg.lose_unsynced_dirents)
self.noteDirent(.rename, new_dir, new_sub_path, old_sub_path, if (self.exists(new_dir, new_sub_path)) new_sub_path else null);
try self.backend_vtable.dirRename(ud, old_dir, old_sub_path, new_dir, new_sub_path);
self.fireHook();
return;
},
.frozen, .crash_now => return injected,
}
}
};
/// Payload-derived, pure function of the byte offset (faulty_file.zig §B): every
/// retry corrupts identically and the mask is always nonzero, so it always flips.
fn corruptionMask(off: u64) u8 {
const h = std.hash.Wyhash.hash(0, std.mem.asBytes(&off));
return @truncate(1 | (h & 7));
}
test "faulty_dir: a healthy schedule delegates every op unchanged" {
const t = std.testing;
var tmp = t.tmpDir(.{});
defer tmp.cleanup();
var prng = std.Random.DefaultPrng.init(1);
var faulty = FaultyDir.init(t.io, prng.random(), .{}); // no crash
faulty.install();
defer FaultyDir.uninstall();
const io = faulty.io();
var file = try tmp.dir.createFile(io, "f", .{ .read = false, .truncate = true, .resolve_beneath = true });
try file.writePositionalAll(io, "hello world", 0);
try file.sync(io);
file.close(io);
try t.expect(!faulty.crashed);
const back = try tmp.dir.readFileAlloc(t.io, "f", t.allocator, .limited(64));
defer t.allocator.free(back);
try t.expectEqualStrings("hello world", back);
}
test "faulty_dir: the crash op errors and freezes every later mutation" {
const t = std.testing;
var tmp = t.tmpDir(.{});
defer tmp.cleanup();
var prng = std.Random.DefaultPrng.init(2);
// Crash at the first mutating op: the very first createFile fails, and the
// decorator latches so the follow-up delete is a no-op error too.
var faulty = FaultyDir.init(t.io, prng.random(), .{ .crash_at_op = 1 });
faulty.install();
defer FaultyDir.uninstall();
const io = faulty.io();
try t.expectError(error.AccessDenied, tmp.dir.createFile(io, "f", .{ .truncate = true, .resolve_beneath = true }));
try t.expect(faulty.crashed);
try t.expectEqual(@as(u64, 1), faulty.crash_op_index);
// Frozen: a subsequent delete cannot move a byte.
try t.expectError(error.AccessDenied, tmp.dir.deleteFile(io, "anything"));
}
test "faulty_dir: lose_unsynced discards the tail past the last fsync, and says it engaged" {
const t = std.testing;
var tmp = t.tmpDir(.{});
defer tmp.cleanup();
var prng = std.Random.DefaultPrng.init(3);
var faulty = FaultyDir.init(t.io, prng.random(), .{ .lose_unsynced = true });
faulty.install();
defer FaultyDir.uninstall();
const io = faulty.io();
var f = try tmp.dir.createFile(io, "f", .{ .truncate = true, .resolve_beneath = true });
try f.writePositionalAll(io, "durable-", 0);
try f.sync(io);
try f.writePositionalAll(io, "lost", 8);
f.close(io);
// `dirty_closes` is what a sweep asserts on: without it, a run whose model
// silently stopped engaging is indistinguishable from one whose fsync
// happened to cover everything.
try t.expectEqual(@as(u64, 1), faulty.dirty_closes);
const back = try tmp.dir.readFileAlloc(t.io, "f", t.allocator, .limited(64));
defer t.allocator.free(back);
try t.expectEqualStrings("durable-", back);
}
test "faulty_dir: a torn write leaves a strict prefix, matching neither image" {
const t = std.testing;
var tmp = t.tmpDir(.{});
defer tmp.cleanup();
// First lay down the pre-image with a healthy decorator.
{
var prng0 = std.Random.DefaultPrng.init(0);
var healthy = FaultyDir.init(t.io, prng0.random(), .{});
healthy.install();
defer FaultyDir.uninstall();
const hio = healthy.io();
var f = try tmp.dir.createFile(hio, "f", .{ .truncate = true, .resolve_beneath = true });
try f.writePositionalAll(hio, "AAAAAAAAAAAAAAAA", 0);
try f.sync(hio);
f.close(hio);
}
// Now crash mid-write of the post-image: the file already exists, so open +
// positional overwrite tears a prefix of the new bytes over the old.
var prng = std.Random.DefaultPrng.init(7);
var faulty = FaultyDir.init(t.io, prng.random(), .{ .crash_at_op = 1, .torn_write = true });
faulty.install();
defer FaultyDir.uninstall();
const io = faulty.io();
var f = try tmp.dir.openFile(io, "f", .{ .mode = .write_only, .resolve_beneath = true });
try t.expectError(error.AccessDenied, f.writePositionalAll(io, "BBBBBBBBBBBBBBBB", 0));
f.close(io);
const back = try tmp.dir.readFileAlloc(t.io, "f", t.allocator, .limited(64));
defer t.allocator.free(back);
// A strict, non-empty prefix of B's overwrote A's: neither all-A nor all-B.
try t.expect(back.len == 16);
try t.expect(std.mem.indexOfScalar(u8, back, 'B') != null); // some new bytes landed
try t.expect(std.mem.indexOfScalar(u8, back, 'A') != null); // but not all of them
}
test "faulty_dir: a horizon stamp torn at the store header leaves a store that verifies" {
const t = std.testing;
const zit = @import("zit");
const logstore = zit.logstore;
const gpa = t.allocator;
const blob = "a blob the torn stamp must not lose\n";
const blob_hash = "0123456789abcdef0123456789abcdef01234567";
var cone = try zit.cone.Cone.initHorizon(gpa, &.{}, &.{}, .{ .hash = blob_hash, .rule = .descendants });
defer cone.deinit(gpa);
// Each seed tears the 64-byte header write at a different length; enough
// seeds reach both the version byte and the crc bytes.
var version_torn = false;
var crc_torn = false;
var seed: u64 = 0;
while (seed < 64) : (seed += 1) {
var tmp = t.tmpDir(.{});
defer tmp.cleanup();
{
const raw = try tmp.dir.createFile(t.io, zit.repo_filename, .{ .read = true });
defer raw.close(t.io);
var fs = zit.storage.FileStorage.init(t.io, raw, 0);
var log = try logstore.LogStore.open(gpa, fs.storageFile(), 0);
defer log.deinit(gpa);
var digest: [20]u8 = undefined;
_ = try std.fmt.hexToBytes(&digest, blob_hash);
_ = try log.put(&digest, blob);
try log.commit();
}
var before: [logstore.header_len]u8 = undefined;
_ = try tmp.dir.readFile(t.io, zit.repo_filename, &before);
var prng = std.Random.DefaultPrng.init(seed);
var faulty = FaultyDir.init(t.io, prng.random(), .{ .crash_at_op = 1, .torn_write = true });
faulty.install();
zit.repocache.writeStoreCone(gpa, faulty.io(), tmp.dir, &cone);
FaultyDir.uninstall();
// The stamp failed, so no horizon policy reached disk.
try t.expectError(error.FileNotFound, tmp.dir.access(t.io, zit.repocache.store_cone_filename, .{}));
const raw = try tmp.dir.openFile(t.io, zit.repo_filename, .{});
defer raw.close(t.io);
const size = (try raw.stat(t.io)).size;
var fs = zit.storage.FileStorage.init(t.io, raw, size);
var after: [logstore.header_len]u8 = undefined;
_ = try fs.storageFile().pread(0, &after);
if (after[4] != before[4]) version_torn = true;
if (!std.mem.eql(u8, after[60..64], before[60..64])) crc_torn = true;
var verifier = try logstore.FrameVerifier.init(fs.storageFile());
try t.expect(verifier.next() == null);
var log = try logstore.LogStore.openReadOnly(gpa, fs.storageFile());
defer log.deinit(gpa);
try t.expectEqual(@as(u64, 1), log.artifactCount());
}
try t.expect(version_torn and crc_torn);
}