Skip to content
zit Artifact [22e41b391a]
Login

Artifact [22e41b391a]

Check-ins Using Download Hex

Artifact 22e41b391ae0d3b76f4869b4d924a8a2ef0b76a4f465a1bb649f34fca3fe97f8:

#!/usr/bin/env bash
#
# soak.sh — out-of-band deep DST + fuzz soak runner.
# Requires bash (uses the SECONDS builtin for the per-target wall-clock budget).
#
# The pre-commit gate and CI run these harnesses shallow, for REGRESSION
# DETECTION: `zig build dst` sweeps a few seeds x a few thousand steps, and CI
# fuzzes one seed x 100 rounds. Neither is a bug hunt. This script is the bug
# hunt: the same harnesses at nightly depth, on a ReleaseSafe build (asserts
# ON), so machine-hours — not new invariants — do the finding. Every harness
# comment in the tree that defers depth to "an out-of-band runner" means this.
#
# It NEVER auto-files and NEVER auto-fixes. On any failure it appends the
# harness's own verbatim `reproduce with seed=...` line (reshaped into a
# runnable command) to a dated log and drops a flag file the operator's next
# session sees -- then KEEPS GOING. A failing fuzz target is abandoned (a
# crash repeats; grinding it finds nothing new) and the sweep moves to the
# next target; a failing DST phase moves to the next phase. Only a failure
# that leaves nothing to run ends the sweep early. At the end every finding is
# listed and the exit is non-zero if there was one. Stopping at the first
# finding hid the rest: one recurring crash in the sixteenth target left the
# twelve after it unsoaked for four nights. Triage is the operator's; a soak
# finding is a P6 GO/NO-GO input, not a footnote.
#
# ---------------------------------------------------------------------------
# Usage
#   scripts/soak.sh                 # full nightly (see defaults below)
#   DST_SEEDS=3 DST_STEPS=500 \
#     FUZZ_TARGET_BUDGET=6 FUZZ_SEEDS=3 scripts/soak.sh   # bounded smoke
#
# Config (all env-overridable; defaults are the nightly budget):
#   DST_SEEDS=200          seeds swept by the repostate+sync DST
#   DST_STEPS=10000        steps per DST seed
#   DST_FAST=1             also sweep the DST on a ReleaseFast build, the
#                          build that ships, where only the simulation's own
#                          oracle can catch a bug (no asserts); 0 skips
#   FUZZ_TARGET_BUDGET=300 wall-clock seconds granted to EACH fuzz target
#   FUZZ_SEEDS=20          fresh-seed invocations per target (sets the slice:
#                          slice = FUZZ_TARGET_BUDGET / FUZZ_SEEDS seconds)
#   FUZZ_BASE_ROUNDS=16    starting --rounds; auto-grows (x2) to fill a slice
#   FUZZ_MAX_ROUNDS=100000000  growth cap (one invocation never exceeds it)
#   ZIT_SOAK_DIR=$HOME/.zit-soak   log + flag directory (kept OUT of the repo
#                          so a soak run never shows up in `fossil status`)
#
# Depth-by-time, not by flat rounds: a flat --rounds would run
# artifact_parse_garbage in seconds while logstore_pack_same_path_round_trip
# (~276x heavier per round, see src/fuzz_tests.zig:63-78) ran for hours. Each
# target instead gets an equal wall-clock budget; --rounds is grown per target
# from measured speed to fill each slice, so cheap targets sweep millions of
# rounds and heavy targets sweep thousands — the cost ratios honor themselves,
# with no second per-target table to drift. The target LIST comes from the
# runner's own usage output (the src/.github/workflows/ci.yml:146-159 pattern),
# so it cannot drift either.
#
# ---------------------------------------------------------------------------
# Operator install (NOT done by this script — installing the schedule and
# running the first full nightly is an operator action):
#
#   macOS launchd (the mac-mini runner) — write to
#   ~/Library/LaunchAgents/group.zit.soak.plist then `launchctl load` it:
#
#     <?xml version="1.0" encoding="UTF-8"?>
#     <!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN"
#       "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
#     <plist version="1.0"><dict>
#       <key>Label</key><string>group.zit.soak</string>
#       <key>ProgramArguments</key>
#         <array>
#           <string>/Users/dmestas/zit/scripts/soak.sh</string>
#         </array>
#       <key>StartCalendarInterval</key>
#         <dict><key>Hour</key><integer>3</integer>
#               <key>Minute</key><integer>0</integer></dict>
#       <key>StandardOutPath</key><string>/tmp/zit-soak.out</string>
#       <key>StandardErrorPath</key><string>/tmp/zit-soak.err</string>
#     </dict></plist>
#
#     launchctl load ~/Library/LaunchAgents/group.zit.soak.plist
#
#   cron equivalent (3am daily):
#     0 3 * * * cd /Users/dmestas/zit && ./scripts/soak.sh
#
# After a run, check for the flag file printed on failure; triage, then delete
# the flag once the finding is dispositioned. A clean run leaves any prior flag
# untouched — the soak never clears a failure the operator has not yet seen.
#
set -eu

# --- locate the repo root (parent of this script's dir) and build there ------
script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
repo_root=$(CDPATH= cd -- "${script_dir}/.." && pwd)
cd "${repo_root}"

# --- config ------------------------------------------------------------------
DST_SEEDS=${DST_SEEDS:-200}
DST_STEPS=${DST_STEPS:-10000}
FUZZ_TARGET_BUDGET=${FUZZ_TARGET_BUDGET:-300}
FUZZ_SEEDS=${FUZZ_SEEDS:-20}
FUZZ_BASE_ROUNDS=${FUZZ_BASE_ROUNDS:-16}
FUZZ_MAX_ROUNDS=${FUZZ_MAX_ROUNDS:-100000000}
ZIT_SOAK_DIR=${ZIT_SOAK_DIR:-${HOME}/.zit-soak}

# ReleaseSafe: the whole point is asserts ON at depth. Never change this.
OPT=--release=safe

# per-target slice: an equal share of the target budget across FUZZ_SEEDS
# fresh-seed invocations, floored at 1s so integer SECONDS math stays sane.
slice=$(( FUZZ_TARGET_BUDGET / FUZZ_SEEDS ))
[ "${slice}" -lt 1 ] && slice=1

mkdir -p "${ZIT_SOAK_DIR}"
stamp=$(date +%Y%m%d)
log="${ZIT_SOAK_DIR}/soak-${stamp}.log"
flag="${ZIT_SOAK_DIR}/SOAK-FAILED"
run_stderr="${ZIT_SOAK_DIR}/.soak-run.stderr"

started=$(date -u +%Y-%m-%dT%H:%M:%SZ)
printf '=== soak start %s (dst %sx%s, fuzz %ss/target) ===\n' \
  "${started}" "${DST_SEEDS}" "${DST_STEPS}" "${FUZZ_TARGET_BUDGET}" >>"${log}"

# fail PHASE REPRODUCE_CMD — append the verbatim reproduce line, raise the flag,
# and count the finding. RETURNS: the caller decides whether anything is left to
# run. Never files, never fixes.
findings=0
failed_phases=""
fail() {
  _phase=$1
  _cmd=$2
  _when=$(date -u +%Y-%m-%dT%H:%M:%SZ)
  {
    printf 'FAIL %s [%s]\n' "${_phase}" "${_when}"
    printf 'reproduce: %s\n' "${_cmd}"
    printf -- '--- captured harness output (tail) ---\n'
    tail -n 20 "${run_stderr}" 2>/dev/null || true
    printf -- '---\n'
  } >>"${log}"
  {
    printf 'zit soak FAILED %s in %s\n' "${_when}" "${_phase}"
    printf 'reproduce: %s\n' "${_cmd}"
    printf 'log: %s\n' "${log}"
  } >>"${flag}"
  printf 'soak: FAILED %s. reproduce logged to %s ; flag at %s\n' \
    "${_phase}" "${log}" "${flag}" >&2
  findings=$(( findings + 1 ))
  failed_phases="${failed_phases} ${_phase}"
}

# finish — close the log and exit: CLEAN and 0, or every finding and 1.
finish() {
  _finished=$(date -u +%Y-%m-%dT%H:%M:%SZ)
  if [ "${findings}" -eq 0 ]; then
    printf '=== soak CLEAN %s ===\n' "${_finished}" >>"${log}"
    printf 'soak: CLEAN. log: %s\n' "${log}" >&2
    exit 0
  fi
  printf '=== soak FAILED %s: %s finding(s):%s ===\n' "${_finished}" "${findings}" "${failed_phases}" >>"${log}"
  printf 'SOAK FAILED: %s finding(s):%s. reproduce lines in %s ; flag at %s\n' \
    "${findings}" "${failed_phases}" "${log}" "${flag}" >&2
  exit 1
}

# locate_exe NAME — echo the newest built executable NAME under the cache.
# The soak runs the built binaries DIRECTLY, not through `zig build ... --`,
# for two reasons: `zig build` captures and DISCARDS a failed run's child
# stderr (so the harness's `reproduce with seed=...` line would be lost, the one
# thing this runner exists to record), and a direct exec skips the per-round
# build-graph check. `zig build` is used ONLY to compile.
# The compile cache is `.zig-cache` in a checkout and ZIG_LOCAL_CACHE_DIR on
# the CI runners, which keep it outside the workspace. An empty find MUST NOT
# reach `ls`: `xargs ls -t` with nothing to list lists the current directory,
# and the newest entry of the repo root ("src") was once run as the harness.
locate_exe() {
  local cache="${ZIG_LOCAL_CACHE_DIR:-.zig-cache}"
  local found
  found=$(find "${cache}/o" -type f -name "$1" -perm -u+x 2>/dev/null -print0 \
    | xargs -0 -r ls -t 2>/dev/null | head -n 1)
  [ -n "${found}" ] && [ -x "${found}" ] || return 1
  printf '%s' "${found}"
}

# --- phase 1: the DST sweep --------------------------------------------------
printf 'soak: dst --seeds %s --steps %s (ReleaseSafe)\n' "${DST_SEEDS}" "${DST_STEPS}" >&2
# Compile the ReleaseSafe dst exe (the trivial run may exit non-zero if an
# invariant is already broken; the COMPILE is what we need, so ignore its exit).
zig build dst "${OPT}" -- --seeds 1 --steps 1 >/dev/null 2>&1 || true
dst_exe=$(locate_exe dst) || dst_exe=""
if [ -z "${dst_exe}" ]; then
  fail "dst-build" "zig build dst ${OPT}   # compile failed"
elif "${dst_exe}" --seeds "${DST_SEEDS}" --steps "${DST_STEPS}" \
     >/dev/null 2>"${run_stderr}"; then
  printf 'soak: dst clean (%s seeds x %s steps)\n' "${DST_SEEDS}" "${DST_STEPS}" >>"${log}"
else
  # dst prints `reproduce with seed=N` (decimal) for both the repostate and the
  # sync sweep. The replay needs the same --steps, which the line omits.
  seed=$(grep -o 'reproduce with seed=[0-9]*' "${run_stderr}" | tail -n 1 | sed 's/.*=//')
  [ -z "${seed}" ] && seed='<unknown>'
  fail "dst" "zig build dst ${OPT} -- --seed ${seed} --steps ${DST_STEPS}"
fi

# --- phase 1b: the DST sweep on the production build -------------------------
# Production is ReleaseFast, with every assert and safety check stripped. One
# class of bug shows only there: behaviour that changes when a check is
# absent, or undefined behaviour the optimizer exploits. The simulation's own
# oracle compares replayed states rather than relying on asserts, so the
# same sweep on a ReleaseFast build covers the build that ships. DST_FAST=0
# skips it.
if [ "${DST_FAST:-1}" != "0" ]; then
  printf 'soak: dst --seeds %s --steps %s (ReleaseFast, the production build)\n' "${DST_SEEDS}" "${DST_STEPS}" >&2
  zig build dst --release=fast -- --seeds 1 --steps 1 >/dev/null 2>&1 || true
  dst_fast_exe=$(locate_exe dst) || dst_fast_exe=""
  if [ -z "${dst_fast_exe}" ]; then
    fail "dst-fast-build" "zig build dst --release=fast   # compile failed"
  elif "${dst_fast_exe}" --seeds "${DST_SEEDS}" --steps "${DST_STEPS}" \
       >/dev/null 2>"${run_stderr}"; then
    printf 'soak: dst clean on ReleaseFast (%s seeds x %s steps)\n' "${DST_SEEDS}" "${DST_STEPS}" >>"${log}"
  else
    seed=$(grep -o 'reproduce with seed=[0-9]*' "${run_stderr}" | tail -n 1 | sed 's/.*=//')
    [ -z "${seed}" ] && seed='<unknown>'
    fail "dst-fast" "zig build dst --release=fast -- --seed ${seed} --steps ${DST_STEPS}"
  fi
fi

# --- phase 2: the fuzz sweep -------------------------------------------------
# Enumerate from the runner's own usage output (ci.yml:146-159 pattern), which
# also compiles the ReleaseSafe fuzz exe.
targets=$( (zig build fuzz "${OPT}" 2>&1 || true) \
  | sed -n '/^targets:/,$p' | grep -E '^  [a-z0-9_]+$' | tr -d ' ' )
# Neither of these leaves anything to run, so the sweep ends at them.
if [ -z "${targets}" ]; then
  fail "fuzz-enumerate" "zig build fuzz ${OPT}   # usage format changed; sweep is vacuous"
  finish
fi
fuzz_exe=$(locate_exe fuzz) || fuzz_exe=""
if [ -z "${fuzz_exe}" ]; then
  fail "fuzz-build" "zig build fuzz ${OPT}   # compile failed"
  finish
fi

for target in ${targets}; do
  printf 'soak: fuzz %s (budget %ss, slice %ss)\n' "${target}" "${FUZZ_TARGET_BUDGET}" "${slice}" >&2
  rounds=${FUZZ_BASE_ROUNDS}
  t_start=${SECONDS}
  target_failed=0
  while [ $(( SECONDS - t_start )) -lt "${FUZZ_TARGET_BUDGET}" ]; do
    i_start=${SECONDS}
    # A fresh explicit seed per invocation, drawn here (8 random bytes -> u64),
    # NOT left to the runner's csprng. The runner prints `reproduce with
    # seed=0x...` before a run, but its buffered stderr is DROPPED on the error
    # return path (Zig 0.16 Io): a crashing target emits only `error: X`, not
    # its seed. Owning the seed makes the logged reproduce line exact anyway.
    seed="0x$(od -An -N8 -tx1 /dev/urandom | tr -d ' \n')"
    if "${fuzz_exe}" "${target}" --seed "${seed}" --rounds "${rounds}" \
         >/dev/null 2>"${run_stderr}"; then
      :
    else
      fail "fuzz:${target}" "zig build fuzz ${OPT} -- ${target} --seed ${seed} --rounds ${rounds}"
      target_failed=1
      break
    fi
    i_dt=$(( SECONDS - i_start ))
    # Grow rounds (x2) toward filling a slice while an invocation still finishes
    # under it; measured, not guessed — the same discipline the harness uses.
    # Start small (FUZZ_BASE_ROUNDS) so even the heaviest target's first
    # invocation fits a slice, then settle where i_dt ~= slice. There is no
    # mid-invocation kill, so a settled invocation may overshoot the slice by up
    # to itself; the target budget is therefore a soft bound, honest for a soak.
    if [ "${i_dt}" -lt "${slice}" ] && [ "${rounds}" -lt "${FUZZ_MAX_ROUNDS}" ]; then
      rounds=$(( rounds * 2 ))
      [ "${rounds}" -gt "${FUZZ_MAX_ROUNDS}" ] && rounds=${FUZZ_MAX_ROUNDS}
    fi
  done
  if [ "${target_failed}" -eq 0 ]; then
    printf 'soak: fuzz %s clean (final rounds=%s)\n' "${target}" "${rounds}" >>"${log}"
  fi
done

finish