#!/usr/bin/env bash
#
# soak.sh — out-of-band deep DST + fuzz soak runner.
# Requires bash (uses the SECONDS builtin for the per-target wall-clock budget).
#
# The pre-commit gate and CI run these harnesses shallow, for REGRESSION
# DETECTION: `zig build dst` sweeps a few seeds x a few thousand steps, and CI
# fuzzes one seed x 100 rounds. Neither is a bug hunt. This script is the bug
# hunt: the same harnesses at nightly depth, on a ReleaseSafe build (asserts
# ON), so machine-hours — not new invariants — do the finding. Every harness
# comment in the tree that defers depth to "an out-of-band runner" means this.
#
# It NEVER auto-files and NEVER auto-fixes. On any failure it appends the
# harness's own verbatim `reproduce with seed=...` line (reshaped into a
# runnable command) to a dated log and drops a flag file the operator's next
# session sees -- then KEEPS GOING. A failing fuzz target is abandoned (a
# crash repeats; grinding it finds nothing new) and the sweep moves to the
# next target; a failing DST phase moves to the next phase. Only a failure
# that leaves nothing to run ends the sweep early. At the end every finding is
# listed and the exit is non-zero if there was one. Stopping at the first
# finding hid the rest: one recurring crash in the sixteenth target left the
# twelve after it unsoaked for four nights. Triage is the operator's; a soak
# finding is a P6 GO/NO-GO input, not a footnote.
#
# ---------------------------------------------------------------------------
# Usage
# scripts/soak.sh # full nightly (see defaults below)
# DST_SEEDS=3 DST_STEPS=500 \
# FUZZ_TARGET_BUDGET=6 FUZZ_SEEDS=3 scripts/soak.sh # bounded smoke
#
# Config (all env-overridable; defaults are the nightly budget):
# DST_SEEDS=200 seeds swept by the repostate+sync DST
# DST_STEPS=10000 steps per DST seed
# DST_FAST=1 also sweep the DST on a ReleaseFast build, the
# build that ships, where only the simulation's own
# oracle can catch a bug (no asserts); 0 skips
# HORIZON_SEEDS=800 further seeds, after DST_SEEDS, swept by the
# horizon sweep alone (seconds a seed, where the
# full DST is minutes); 0 skips
# FUZZ_TARGET_BUDGET=300 wall-clock seconds granted to EACH fuzz target
# FUZZ_SEEDS=20 fresh-seed invocations per target (sets the slice:
# slice = FUZZ_TARGET_BUDGET / FUZZ_SEEDS seconds)
# FUZZ_BASE_ROUNDS=16 starting --rounds; auto-grows (x2) to fill a slice
# FUZZ_MAX_ROUNDS=100000000 growth cap (one invocation never exceeds it)
# ZIT_SOAK_DIR=$HOME/.zit-soak log + flag directory (kept OUT of the repo
# so a soak run never shows up in `fossil status`)
#
# Depth-by-time, not by flat rounds: a flat --rounds would run
# artifact_parse_garbage in seconds while logstore_pack_same_path_round_trip
# (~276x heavier per round, see src/fuzz_tests.zig:63-78) ran for hours. Each
# target instead gets an equal wall-clock budget; --rounds is grown per target
# from measured speed to fill each slice, so cheap targets sweep millions of
# rounds and heavy targets sweep thousands — the cost ratios honor themselves,
# with no second per-target table to drift. The target LIST comes from the
# runner's own usage output (the src/.github/workflows/ci.yml:146-159 pattern),
# so it cannot drift either.
#
# ---------------------------------------------------------------------------
# Operator install (NOT done by this script — installing the schedule and
# running the first full nightly is an operator action):
#
# macOS launchd (the mac-mini runner) — write to
# ~/Library/LaunchAgents/group.zit.soak.plist then `launchctl load` it:
#
# <?xml version="1.0" encoding="UTF-8"?>
# <!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN"
# "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
# <plist version="1.0"><dict>
# <key>Label</key><string>group.zit.soak</string>
# <key>ProgramArguments</key>
# <array>
# <string>/Users/dmestas/zit/scripts/soak.sh</string>
# </array>
# <key>StartCalendarInterval</key>
# <dict><key>Hour</key><integer>3</integer>
# <key>Minute</key><integer>0</integer></dict>
# <key>StandardOutPath</key><string>/tmp/zit-soak.out</string>
# <key>StandardErrorPath</key><string>/tmp/zit-soak.err</string>
# </dict></plist>
#
# launchctl load ~/Library/LaunchAgents/group.zit.soak.plist
#
# cron equivalent (3am daily):
# 0 3 * * * cd /Users/dmestas/zit && ./scripts/soak.sh
#
# After a run, check for the flag file printed on failure; triage, then delete
# the flag once the finding is dispositioned. A clean run leaves any prior flag
# untouched — the soak never clears a failure the operator has not yet seen.
#
set -eu
# --- locate the repo root (parent of this script's dir) and build there ------
script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
repo_root=$(CDPATH= cd -- "${script_dir}/.." && pwd)
cd "${repo_root}"
# --- config ------------------------------------------------------------------
DST_SEEDS=${DST_SEEDS:-200}
DST_STEPS=${DST_STEPS:-10000}
HORIZON_SEEDS=${HORIZON_SEEDS:-800}
FUZZ_TARGET_BUDGET=${FUZZ_TARGET_BUDGET:-300}
FUZZ_SEEDS=${FUZZ_SEEDS:-20}
FUZZ_BASE_ROUNDS=${FUZZ_BASE_ROUNDS:-16}
FUZZ_MAX_ROUNDS=${FUZZ_MAX_ROUNDS:-100000000}
ZIT_SOAK_DIR=${ZIT_SOAK_DIR:-${HOME}/.zit-soak}
# ReleaseSafe: the whole point is asserts ON at depth. Never change this.
OPT=--release=safe
# per-target slice: an equal share of the target budget across FUZZ_SEEDS
# fresh-seed invocations, floored at 1s so integer SECONDS math stays sane.
slice=$(( FUZZ_TARGET_BUDGET / FUZZ_SEEDS ))
[ "${slice}" -lt 1 ] && slice=1
mkdir -p "${ZIT_SOAK_DIR}"
stamp=$(date +%Y%m%d)
log="${ZIT_SOAK_DIR}/soak-${stamp}.log"
flag="${ZIT_SOAK_DIR}/SOAK-FAILED"
run_stderr="${ZIT_SOAK_DIR}/.soak-run.stderr"
started=$(date -u +%Y-%m-%dT%H:%M:%SZ)
printf '=== soak start %s (dst %sx%s, fuzz %ss/target) ===\n' \
"${started}" "${DST_SEEDS}" "${DST_STEPS}" "${FUZZ_TARGET_BUDGET}" >>"${log}"
# keep_stderr PHASE — copy the run's stderr to a dated, non-hidden file beside
# the log. The run file itself is rewritten by the next phase, and the CI
# upload skips hidden files.
keep_stderr() {
cp "${run_stderr}" "${ZIT_SOAK_DIR}/soak-${stamp}-$(printf '%s' "$1" | tr -c 'A-Za-z0-9_.-' '-').stderr" 2>/dev/null || true
}
# failing_seed — the seed a failed DST run names: its `reproduce with seed=N`
# line, else the last `dst: seed N` it began (a crash prints no reproduce
# line), else `<unknown>`.
failing_seed() {
_seed=$(grep -o 'reproduce with seed=[0-9]*' "${run_stderr}" | tail -n 1 | sed 's/.*=//')
[ -z "${_seed}" ] && _seed=$(grep -o '^dst: seed [0-9]*' "${run_stderr}" | tail -n 1 | sed 's/.* //')
[ -z "${_seed}" ] && _seed='<unknown>'
printf '%s' "${_seed}"
}
# fail PHASE REPRODUCE_CMD [STATUS] — append the verbatim reproduce line, raise
# the flag, and count the finding. With STATUS (the run's exit status), also
# the head and tail of the run's stderr: the head names what started, the tail
# what failed, and a crash's report runs longer than its last few lines.
# Without one nothing ran, and the run file is a previous phase's. RETURNS:
# the caller decides whether anything is left to run. Never files, never fixes.
findings=0
failed_phases=""
fail() {
_phase=$1
_cmd=$2
_status=${3:-}
_when=$(date -u +%Y-%m-%dT%H:%M:%SZ)
{
printf 'FAIL %s [%s]\n' "${_phase}" "${_when}"
printf 'reproduce: %s\n' "${_cmd}"
if [ -n "${_status}" ]; then
printf 'exit status: %s\n' "${_status}"
_lines=$(wc -l <"${run_stderr}" 2>/dev/null | tr -d ' ')
if [ "${_lines:-0}" -le 45 ]; then
printf -- '--- captured harness output ---\n'
cat "${run_stderr}" 2>/dev/null || true
else
printf -- '--- captured harness output (head) ---\n'
head -n 5 "${run_stderr}"
printf -- '--- (%s lines omitted) ---\n' "$((_lines - 45))"
printf -- '--- captured harness output (tail) ---\n'
tail -n 40 "${run_stderr}"
fi
printf -- '---\n'
fi
} >>"${log}"
{
printf 'zit soak FAILED %s in %s\n' "${_when}" "${_phase}"
printf 'reproduce: %s\n' "${_cmd}"
printf 'log: %s\n' "${log}"
} >>"${flag}"
printf 'soak: FAILED %s. reproduce logged to %s ; flag at %s\n' \
"${_phase}" "${log}" "${flag}" >&2
findings=$(( findings + 1 ))
failed_phases="${failed_phases} ${_phase}"
}
# finish — close the log and exit: CLEAN and 0, or every finding and 1.
finish() {
_finished=$(date -u +%Y-%m-%dT%H:%M:%SZ)
if [ "${findings}" -eq 0 ]; then
printf '=== soak CLEAN %s ===\n' "${_finished}" >>"${log}"
printf 'soak: CLEAN. log: %s\n' "${log}" >&2
exit 0
fi
printf '=== soak FAILED %s: %s finding(s):%s ===\n' "${_finished}" "${findings}" "${failed_phases}" >>"${log}"
printf 'SOAK FAILED: %s finding(s):%s. reproduce lines in %s ; flag at %s\n' \
"${findings}" "${failed_phases}" "${log}" "${flag}" >&2
exit 1
}
# locate_exe NAME — echo the newest built executable NAME under the cache.
# The soak runs the built binaries DIRECTLY, not through `zig build ... --`,
# for two reasons: `zig build` captures and DISCARDS a failed run's child
# stderr (so the harness's `reproduce with seed=...` line would be lost, the one
# thing this runner exists to record), and a direct exec skips the per-round
# build-graph check. `zig build` is used ONLY to compile.
# The compile cache is `.zig-cache` in a checkout and ZIG_LOCAL_CACHE_DIR on
# the CI runners, which keep it outside the workspace. An empty find MUST NOT
# reach `ls`: `xargs ls -t` with nothing to list lists the current directory,
# and the newest entry of the repo root ("src") was once run as the harness.
locate_exe() {
local cache="${ZIG_LOCAL_CACHE_DIR:-.zig-cache}"
local found
found=$(find "${cache}/o" -type f -name "$1" -perm -u+x 2>/dev/null -print0 \
| xargs -0 -r ls -t 2>/dev/null | head -n 1)
[ -n "${found}" ] && [ -x "${found}" ] || return 1
printf '%s' "${found}"
}
# --- phase 1: the DST sweep --------------------------------------------------
printf 'soak: dst --seeds %s --steps %s (ReleaseSafe)\n' "${DST_SEEDS}" "${DST_STEPS}" >&2
# Compile the ReleaseSafe dst exe (the trivial run may exit non-zero if an
# invariant is already broken; the COMPILE is what we need, so ignore its exit).
zig build dst "${OPT}" -- --seeds 1 --steps 1 >/dev/null 2>&1 || true
dst_exe=$(locate_exe dst) || dst_exe=""
if [ -z "${dst_exe}" ]; then
fail "dst-build" "zig build dst ${OPT} # compile failed"
elif "${dst_exe}" --seeds "${DST_SEEDS}" --steps "${DST_STEPS}" \
>/dev/null 2>"${run_stderr}"; then
keep_stderr dst
printf 'soak: dst clean (%s seeds x %s steps)\n' "${DST_SEEDS}" "${DST_STEPS}" >>"${log}"
else
status=$?
keep_stderr dst
# dst prints `reproduce with seed=N` (decimal) for both the repostate and the
# sync sweep. The replay needs the same --steps, which the line omits.
fail "dst" "zig build dst ${OPT} -- --seed $(failing_seed) --steps ${DST_STEPS}" "${status}"
fi
# --- phase 1b: the DST sweep on the production build -------------------------
# Production is ReleaseFast, with every assert and safety check stripped. One
# class of bug shows only there: behaviour that changes when a check is
# absent, or undefined behaviour the optimizer exploits. The simulation's own
# oracle compares replayed states rather than relying on asserts, so the
# same sweep on a ReleaseFast build covers the build that ships. DST_FAST=0
# skips it.
if [ "${DST_FAST:-1}" != "0" ]; then
printf 'soak: dst --seeds %s --steps %s (ReleaseFast, the production build)\n' "${DST_SEEDS}" "${DST_STEPS}" >&2
zig build dst --release=fast -- --seeds 1 --steps 1 >/dev/null 2>&1 || true
dst_fast_exe=$(locate_exe dst) || dst_fast_exe=""
if [ -z "${dst_fast_exe}" ]; then
fail "dst-fast-build" "zig build dst --release=fast # compile failed"
elif "${dst_fast_exe}" --seeds "${DST_SEEDS}" --steps "${DST_STEPS}" \
>/dev/null 2>"${run_stderr}"; then
keep_stderr dst-fast
printf 'soak: dst clean on ReleaseFast (%s seeds x %s steps)\n' "${DST_SEEDS}" "${DST_STEPS}" >>"${log}"
else
status=$?
keep_stderr dst-fast
fail "dst-fast" "zig build dst --release=fast -- --seed $(failing_seed) --steps ${DST_STEPS}" "${status}"
fi
fi
# --- phase 1c: the horizon sweep past the DST's seeds -------------------------
# Phase 1 runs the horizon sweep on each of its seeds too, but the full DST
# costs minutes a seed and the horizon sweep seconds, so the horizon sweep
# alone carries on from where phase 1 stopped. On the ReleaseSafe exe.
if [ "${HORIZON_SEEDS}" != "0" ] && [ -n "${dst_exe}" ]; then
horizon_first=$((DST_SEEDS + 1))
printf 'soak: horizon --seed %s --seeds %s (ReleaseSafe)\n' "${horizon_first}" "${HORIZON_SEEDS}" >&2
if "${dst_exe}" --seed "${horizon_first}" --seeds "${HORIZON_SEEDS}" --horizon-only \
>/dev/null 2>"${run_stderr}"; then
keep_stderr horizon
printf 'soak: horizon clean (seeds %s..%s)\n' "${horizon_first}" "$((DST_SEEDS + HORIZON_SEEDS))" >>"${log}"
else
status=$?
keep_stderr horizon
fail "horizon" "zig build dst ${OPT} -- --seed $(failing_seed) --horizon-only" "${status}"
fi
fi
# --- phase 2: the fuzz sweep -------------------------------------------------
# Enumerate from the runner's own usage output (ci.yml:146-159 pattern), which
# also compiles the ReleaseSafe fuzz exe.
targets=$( (zig build fuzz "${OPT}" 2>&1 || true) \
| sed -n '/^targets:/,$p' | grep -E '^ [a-z0-9_]+$' | tr -d ' ' )
# Neither of these leaves anything to run, so the sweep ends at them.
if [ -z "${targets}" ]; then
fail "fuzz-enumerate" "zig build fuzz ${OPT} # usage format changed; sweep is vacuous"
finish
fi
fuzz_exe=$(locate_exe fuzz) || fuzz_exe=""
if [ -z "${fuzz_exe}" ]; then
fail "fuzz-build" "zig build fuzz ${OPT} # compile failed"
finish
fi
for target in ${targets}; do
printf 'soak: fuzz %s (budget %ss, slice %ss)\n' "${target}" "${FUZZ_TARGET_BUDGET}" "${slice}" >&2
rounds=${FUZZ_BASE_ROUNDS}
t_start=${SECONDS}
target_failed=0
while [ $(( SECONDS - t_start )) -lt "${FUZZ_TARGET_BUDGET}" ]; do
i_start=${SECONDS}
# A fresh explicit seed per invocation, drawn here (8 random bytes -> u64),
# NOT left to the runner's csprng. The runner prints `reproduce with
# seed=0x...` before a run, but its buffered stderr is DROPPED on the error
# return path (Zig 0.16 Io): a crashing target emits only `error: X`, not
# its seed. Owning the seed makes the logged reproduce line exact anyway.
seed="0x$(od -An -N8 -tx1 /dev/urandom | tr -d ' \n')"
if "${fuzz_exe}" "${target}" --seed "${seed}" --rounds "${rounds}" \
>/dev/null 2>"${run_stderr}"; then
:
else
status=$?
# A clean invocation's stderr says nothing; the failing one's is kept.
keep_stderr "fuzz-${target}"
fail "fuzz:${target}" "zig build fuzz ${OPT} -- ${target} --seed ${seed} --rounds ${rounds}" "${status}"
target_failed=1
break
fi
i_dt=$(( SECONDS - i_start ))
# Grow rounds (x2) toward filling a slice while an invocation still finishes
# under it; measured, not guessed — the same discipline the harness uses.
# Start small (FUZZ_BASE_ROUNDS) so even the heaviest target's first
# invocation fits a slice, then settle where i_dt ~= slice. There is no
# mid-invocation kill, so a settled invocation may overshoot the slice by up
# to itself; the target budget is therefore a soft bound, honest for a soak.
if [ "${i_dt}" -lt "${slice}" ] && [ "${rounds}" -lt "${FUZZ_MAX_ROUNDS}" ]; then
rounds=$(( rounds * 2 ))
[ "${rounds}" -gt "${FUZZ_MAX_ROUNDS}" ] && rounds=${FUZZ_MAX_ROUNDS}
fi
done
if [ "${target_failed}" -eq 0 ]; then
printf 'soak: fuzz %s clean (final rounds=%s)\n' "${target}" "${rounds}" >>"${log}"
fi
done
finish