#!/bin/sh
# The fossil -> zit -> fossil round trip, as a differential harness.
#
# The target invariant: a corpus that goes ORIGIN -> zit -> a stock `fossil
# clone` comes back with the SAME ARTIFACT SET, byte for byte, and a repository
# a stock fossil calls intact. The sync protocol is the transport both ways, so
# nothing here rewrites a corpus — a difference means a real defect, not a
# conversion artifact.
#
# What it compares, in the order a failure is worth reading:
# 1. artifact sets -- the invariant itself, by uuid, both directions
# 2. test-integrity -- stock fossil's own verdict on the result
# 3. event counts -- ci/e/f/g/t/w, so a lost technote or forum post is
# named rather than hidden inside "same artifacts"
# 4. the shun table -- a takedown record that does not survive is a
# SAFETY defect, not a cosmetic one: the clone would
# re-accept suppressed content on its next sync
# 5. phantoms -- a count that GREW means a reference lost its
# referent somewhere in the loop
#
# Usage: scripts/roundtrip-fossil.sh ORIGIN.fossil [WORKDIR]
#
# ORIGIN.fossil is any fossil repository. WORKDIR defaults to a fresh mktemp
# directory and is left behind on failure so the difference can be examined;
# on success it is removed unless KEEP=1.
#
# Requires: a built ./zig-out/bin/zit, and `fossil` on PATH.
set -eu
usage() {
echo "usage: $0 ORIGIN.fossil [WORKDIR]" >&2
exit 2
}
[ $# -ge 1 ] || usage
ORIGIN=$1
[ -f "$ORIGIN" ] || { echo "$0: no such repository: $ORIGIN" >&2; exit 2; }
ORIGIN=$(cd "$(dirname "$ORIGIN")" && pwd)/$(basename "$ORIGIN")
ROOT=$(cd "$(dirname "$0")/.." && pwd)
ZIT=$ROOT/zig-out/bin/zit
[ -x "$ZIT" ] || { echo "$0: build zit first (zig build): $ZIT" >&2; exit 2; }
command -v fossil >/dev/null || { echo "$0: fossil is not on PATH" >&2; exit 2; }
WORK=${2:-$(mktemp -d)}
mkdir -p "$WORK"
WORK=$(cd "$WORK" && pwd)
# Give fossil its OWN global-config database for this run. Two reasons, and
# both have bitten: the harness must not register throwaway clones in the
# operator's real ~/.fossil, and every `fossil` process here contends on that
# one global database -- a server still holding it makes the next `fossil
# clone` die with "database is locked" while the transfer itself was fine.
# Isolated, a leaked server from an earlier run cannot poison this one either.
FOSSIL_HOME=$WORK
export FOSSIL_HOME
# One free loopback port. Racy in principle; in practice the server binds
# immediately below and a collision fails loudly rather than silently passing.
port_free() {
python3 -c 'import socket;s=socket.socket();s.bind(("127.0.0.1",0));print(s.getsockname()[1]);s.close()'
}
SERVER_PID=
cleanup() {
[ -n "$SERVER_PID" ] && kill "$SERVER_PID" 2>/dev/null
return 0
}
trap cleanup EXIT INT TERM
fail() {
echo >&2
echo "ROUND TRIP FAILED: $1" >&2
echo " work directory kept for examination: $WORK" >&2
exit 1
}
step() { echo "== $*"; }
# --- 1. ORIGIN facts -------------------------------------------------------
step "reading the origin"
fossil sqlite3 --readonly -R "$ORIGIN" \
"select uuid from blob where size>=0 order by uuid;" > "$WORK/origin.uuids"
fossil sqlite3 --readonly -R "$ORIGIN" \
"select uuid from blob where size<0 order by uuid;" > "$WORK/origin.phantoms"
fossil sqlite3 --readonly -R "$ORIGIN" \
"select uuid from shun order by uuid;" > "$WORK/origin.shun" 2>/dev/null || : > "$WORK/origin.shun"
fossil sqlite3 --readonly -R "$ORIGIN" \
"select type, count(*) from event group by type order by type;" > "$WORK/origin.events"
echo " $(wc -l < "$WORK/origin.uuids" | tr -d ' ') artifacts," \
"$(wc -l < "$WORK/origin.phantoms" | tr -d ' ') phantoms," \
"$(wc -l < "$WORK/origin.shun" | tr -d ' ') shun rows"
# --- 2. ORIGIN -> zit ------------------------------------------------------
step "serving the origin with stock fossil, cloning it into zit"
PORT=$(port_free)
fossil server "$ORIGIN" --port "$PORT" > "$WORK/fossil-serve.log" 2>&1 &
SERVER_PID=$!
sleep 2
rm -rf "$WORK/zit"
"$ZIT" clone "http://127.0.0.1:$PORT/" "$WORK/zit" > "$WORK/zit-clone.log" 2>&1 \
|| fail "zit clone of the origin failed (see $WORK/zit-clone.log)"
kill "$SERVER_PID" 2>/dev/null; wait "$SERVER_PID" 2>/dev/null || :; SERVER_PID=
sed 's/^/ /' "$WORK/zit-clone.log"
# --- 3. zit -> a stock fossil ---------------------------------------------
step "serving zit, cloning it back with stock fossil"
PORT=$(port_free)
# exec, so $! is the server itself: killing a subshell that merely waits on
# it leaves the server running.
( cd "$WORK/zit" && exec "$ZIT" serve --port "$PORT" ) > "$WORK/zit-serve.log" 2>&1 &
SERVER_PID=$!
sleep 2
rm -f "$WORK/back.fossil"
# `fossil clone` registers the new repository in ~/.fossil, the GLOBAL config
# database -- which step 2's `fossil server` also holds open. Signalling that
# server is not enough: until it is reaped the global database stays locked and
# this clone dies with "database is locked" while the transfer itself was fine.
# That is why step 2 ends in `wait`, and why a failure here is worth reading as
# a lock problem before it is read as a protocol problem.
fossil clone "http://127.0.0.1:$PORT/" "$WORK/back.fossil" > "$WORK/fossil-clone.log" 2>&1 \
|| fail "the fossil clone of zit serve failed (see $WORK/fossil-clone.log)"
kill "$SERVER_PID" 2>/dev/null; wait "$SERVER_PID" 2>/dev/null || :; SERVER_PID=
# --- 4. the comparisons ----------------------------------------------------
step "comparing"
fossil sqlite3 --readonly -R "$WORK/back.fossil" \
"select uuid from blob where size>=0 order by uuid;" > "$WORK/back.uuids"
fossil sqlite3 --readonly -R "$WORK/back.fossil" \
"select uuid from blob where size<0 order by uuid;" > "$WORK/back.phantoms"
fossil sqlite3 --readonly -R "$WORK/back.fossil" \
"select uuid from shun order by uuid;" > "$WORK/back.shun" 2>/dev/null || : > "$WORK/back.shun"
fossil sqlite3 --readonly -R "$WORK/back.fossil" \
"select type, count(*) from event group by type order by type;" > "$WORK/back.events"
status=0
# 4a. The invariant. Reported in BOTH directions: "lost" and "gained" are
# different defects and a symmetric diff count would hide which happened.
#
# An artifact the ORIGIN SHUNS is subtracted first. Shunning is an order not
# to serve content, and zit obeys it — so a shunned artifact that the origin
# still physically holds (fossil keeps the blob until a rebuild scrubs it)
# is EXPECTED to be absent downstream. Comparing the raw sets would report
# the harness's own correct behaviour as a defect, and worse, would train
# the reader to expect a red line here.
comm -23 "$WORK/origin.uuids" "$WORK/origin.shun" > "$WORK/origin.expected"
suppressed=$(comm -12 "$WORK/origin.uuids" "$WORK/origin.shun" | wc -l | tr -d ' ')
if cmp -s "$WORK/origin.expected" "$WORK/back.uuids"; then
note=
[ "$suppressed" -gt 0 ] && note=" ($suppressed shunned at the origin, correctly absent)"
echo " artifact sets: IDENTICAL ($(wc -l < "$WORK/back.uuids" | tr -d ' ') artifacts)$note"
else
lost=$(comm -23 "$WORK/origin.expected" "$WORK/back.uuids" | wc -l | tr -d ' ')
gained=$(comm -13 "$WORK/origin.expected" "$WORK/back.uuids" | wc -l | tr -d ' ')
echo " artifact sets: DIFFER -- $lost lost, $gained gained" >&2
comm -23 "$WORK/origin.expected" "$WORK/back.uuids" > "$WORK/lost.uuids"
comm -13 "$WORK/origin.expected" "$WORK/back.uuids" > "$WORK/gained.uuids"
status=1
fi
# 4b. Stock fossil's own verdict on what it received.
if fossil test-integrity "$WORK/back.fossil" > "$WORK/integrity.log" 2>&1; then
echo " test-integrity: $(grep -o '[0-9]* errors' "$WORK/integrity.log" | tail -1)"
grep -q '0 errors' "$WORK/integrity.log" || status=1
else
echo " test-integrity: FAILED (see $WORK/integrity.log)" >&2
status=1
fi
# 4c. Derived events. "Same artifacts" can still mean a lost technote if a
# card stopped crosslinking, so the counts are compared in their own right.
if cmp -s "$WORK/origin.events" "$WORK/back.events"; then
echo " event counts: IDENTICAL ($(tr '\n' ' ' < "$WORK/back.events"))"
else
echo " event counts: DIFFER" >&2
diff "$WORK/origin.events" "$WORK/back.events" >&2 || :
# Name the benign cause when it applies, rather than leaving the reader to
# rediscover it: a shunned CHECK-IN MANIFEST removes its timeline entry
# downstream while the origin's own event row survives until a rebuild, so
# the counts differ for a correct reason. Still reported as a failure --
# the harness cannot tell that case from a real crosslink regression, and
# guessing in favour of "fine" is how a real one would slip through.
[ "$suppressed" -gt 0 ] && echo " note: the origin shuns $suppressed artifact(s) it still holds; a shunned manifest moves these counts legitimately" >&2
status=1
fi
# 4d. The shun table. A lost takedown record is a safety defect: the clone
# will re-accept the suppressed content on its next sync with any peer.
if cmp -s "$WORK/origin.shun" "$WORK/back.shun"; then
echo " shun table: IDENTICAL ($(wc -l < "$WORK/back.shun" | tr -d ' ') rows)"
else
echo " shun table: DIFFER -- origin $(wc -l < "$WORK/origin.shun" | tr -d ' ')," \
"back $(wc -l < "$WORK/back.shun" | tr -d ' ') (SAFETY: suppressed content would return)" >&2
status=1
fi
# 4e. Phantoms. A phantom count that GREW means some reference lost its
# referent in the loop; shunned artifacts are the usual cause.
o_ph=$(wc -l < "$WORK/origin.phantoms" | tr -d ' ')
b_ph=$(wc -l < "$WORK/back.phantoms" | tr -d ' ')
if [ "$o_ph" -eq "$b_ph" ]; then
echo " phantoms: $b_ph (unchanged)"
else
echo " phantoms: $o_ph -> $b_ph" >&2
if [ "$b_ph" -gt "$o_ph" ]; then
comm -13 "$WORK/origin.phantoms" "$WORK/back.phantoms" > "$WORK/new.phantoms"
# Attribute them rather than just counting: a new phantom that IS a
# shunned artifact points at the shun table, not at the artifact path.
shunned=$(comm -12 "$WORK/new.phantoms" "$WORK/origin.shun" | wc -l | tr -d ' ')
echo " $(wc -l < "$WORK/new.phantoms" | tr -d ' ') new," \
"$shunned of them shunned at the origin" >&2
status=1
fi
fi
echo
if [ "$status" -eq 0 ]; then
echo "ROUND TRIP CLEAN"
[ "${KEEP:-0}" = "1" ] || rm -rf "$WORK"
[ "${KEEP:-0}" = "1" ] && echo " work directory kept: $WORK"
exit 0
fi
fail "see the differences above"