#!/bin/sh
# deploy-test.sh — run deploy.sh, quarry-alert.sh and quarry-deploy-watch.sh
# through their paths against temp directories and stubs: no root, no
# host, no network, nothing outside one mktemp directory. Runs on Linux
# and macOS (a perl stand-in for util-linux flock where there is none).
#
# sh scripts/deploy-test.sh prints ok/FAIL per case; exit 1 on any FAIL
# Stub bodies are single-quoted on purpose (they expand when the stub
# runs), and each case line is a deliberate "check && ok || bad".
# shellcheck disable=SC2016,SC2015,SC2030,SC2031,SC2012
set -u
here=$(cd "$(dirname "$0")" && pwd)
T=$(mktemp -d "${TMPDIR:-/tmp}/deploy-test.XXXXXX")
pids=
cleanup() {
for p in $pids; do kill "$p" 2>/dev/null; wait "$p" 2>/dev/null; done
rm -rf "$T"
}
trap cleanup EXIT
fails=0
ok() { echo "ok: $1"; }
bad() { echo "FAIL: $1"; fails=$((fails + 1)); }
H=0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
OTHER=fedcba9876543210fedcba9876543210fedcba9876543210fedcba9876543210
# ---- stubs --------------------------------------------------------------
S=$T/stub
mkdir -p "$S"
C=$T # the current case's directory; stubs write into it
export C
stub() { printf '#!/bin/sh\n%s\n' "$2" >"$S/$1"; chmod +x "$S/$1"; }
if ! command -v flock >/dev/null 2>&1; then
# util-linux `flock -n FD`, via flock(2) on the inherited descriptor.
stub flock '[ "$1" = -n ] || exit 64
exec perl -MFcntl=:flock -e '"'"'open(my $fh, ">&=", $ARGV[0]) or die; flock($fh, LOCK_EX|LOCK_NB) or exit 1'"'"' "$2"'
fi
stub sudo '[ "$1" = -u ] && shift 2; exec "$@"'
stub chown ':'
stub sleep ':'
stub df 'echo "Filesystem 1K-blocks Used Available"; echo "/dev/x 1 1 999999999"'
stub logger '[ "$1" = -t ] && shift 2; echo "$*" >>"$C/journal"'
stub systemctl 'echo "$*" >>"$C/systemctl"; case "$1" in cat) exit 0 ;; esac'
stub systemd-run 'echo "$*" >>"$C/systemd-run"; exit "$(cat "$C/systemd-run.rc" 2>/dev/null || echo 0)"'
# curl: the forge's API answers from files in the case directory, and so
# do its unversioned files: releases/X.Y.Z/FILE is $C/rel/X.Y.Z/FILE.
stub curl 'url=; out=; prev=; for a; do [ "$prev" = -o ] && out=$a; case "$a" in http*) url=$a ;; esac; prev=$a; done
echo "$url" >>"$C/fetched"
body() {
case "$url" in
*/api/v1/r/*/checkin/*) [ -f "$C/checkin.json" ] || exit 22; cat "$C/checkin.json" ;;
*/api/v1/version) echo "{\"ready\":true}" ;;
*/r/store/uv/releases/*) f=$C/rel/${url##*/uv/releases/}; [ -f "$f" ] || exit 22; cat "$f" ;;
# $C/down: the forge stays silent while the 0.2.0 release is installed.
http://127.0.0.1:8950/) [ -f "$C/down" ] && grep -q "quarry 0.2.0" "$C/bin/quarry" && exit 7; echo "quarry 0.1.0
" ;;
esac
}
if [ -n "$out" ]; then body >"$out"; else body; fi
exit 0'
# docker: "builds" into the mounted tree as $C/build says:
# full | partial (no SHA256SUMS, one missing) | none (dies before bin/runner)
stub docker 'w=; prev=; for a; do [ "$prev" = -v ] && case "$a" in *:/src) w=${a%:/src} ;; esac; [ "$prev" = -e ] && case "$a" in VERSION=*) echo "${a#VERSION=}" >"$C/stamp" ;; esac; prev=$a; done
mkdir -p "$w/bin"
printf "#!/bin/sh\necho quarry\n" >"$w/bin/quarry"; printf "#!/bin/sh\n" >"$w/bin/zit"; chmod +x "$w/bin/quarry" "$w/bin/zit"
mode=$(cat "$C/build")
[ "$mode" = none ] && exit 0
mkdir -p "$w/bin/runner"
for f in linux-amd64 linux-arm64 darwin-arm64 darwin-amd64 windows-amd64.exe; do echo "$f" >"$w/bin/runner/quarry-runner-$f"; done
if [ "$mode" = partial ]; then rm "$w/bin/runner/quarry-runner-linux-arm64"; exit 0; fi
cd "$w/bin/runner" && sha256sum quarry-runner-* >SHA256SUMS'
# One case = a fresh directory: a BIN holding a stub zit that opens a
# fixture tree at hash $C/hash, and a published runner set to keep.
#
# The stub answers exactly what zit 0.3.35 prints (pasted from a real
# scratch store): `open` one line, `info current` the full hash FIRST,
# bare `info` the checkout summary with "repository:" first — so a
# script that reads the wrong one fails here too. Anything else it was
# not told about is an error, not a silent pass. The real-zit cases at
# the end hold these fixtures to the binary.
setup() {
C=$T/case.$1
mkdir -p "$C/bin/runner" "$C/tmp" "$C/tree/scripts"
echo old >"$C/bin/runner/SHA256SUMS"
printf '#!/bin/sh\necho old\n' >"$C/bin/quarry"; chmod +x "$C/bin/quarry"
echo "$H" >"$C/hash"
echo full >"$C/build"
echo "go 1.25" >"$C/tree/go.mod"
echo 0.2.0 >"$C/tree/VERSION"
: >"$C/tags"
cp "$here/build-linux.sh" "$here/runner-check.sh" "$here/deploy.sh" "$here/version-stamp.sh" "$C/tree/scripts/"
cat >"$C/bin/zit" <&2; exit 1 ;;
esac
ZIT
chmod +x "$C/bin/zit"
}
deploy() {
(
export PATH="$S:$PATH" C QUARRY_DEPLOY_BIN="$C/bin" QUARRY_DEPLOY_STORE="$C/store" \
QUARRY_DEPLOY_ZITREL="$C/zitrel" QUARRY_DEPLOY_TMP="$C/tmp" \
QUARRY_DEPLOY_LOCK="$C/lock" QUARRY_DEPLOY_STATE="$C/state"
sh "$here/deploy.sh" "$@" >"$C/out" 2>&1
)
rc=$?
}
journal() { grep -q "$1" "$C/journal" 2>/dev/null; }
# ---- --expect -----------------------------------------------------------
setup e12; deploy --dry-run --expect "$(echo "$H" | cut -c1-12)"
[ "$rc" = 0 ] && ok "a 12-character --expect matching trunk deploys" || bad "12-char expect match: rc=$rc $(tail -1 "$C/out")"
setup e64; deploy --dry-run --expect "$H"
[ "$rc" = 0 ] && ok "a full 64-character --expect matching trunk deploys" || bad "64-char expect match: rc=$rc $(tail -1 "$C/out")"
setup m12; deploy --dry-run --expect "$(echo "$OTHER" | cut -c1-12)"
[ "$rc" = 1 ] && journal 'FAILED stage=open .*trunk moved' && ok "a 12-character --expect of another check-in is refused" || bad "12-char moved: rc=$rc"
setup m64; deploy --dry-run --expect "$OTHER"
[ "$rc" = 1 ] && journal 'FAILED stage=open .*trunk moved' && ok "a 64-character --expect of another check-in is refused" || bad "64-char moved: rc=$rc"
setup short; deploy --dry-run --expect 0123456789a
[ "$rc" = 2 ] && ok "an --expect under 12 characters is a usage error" || bad "short expect: rc=$rc"
setup nothex; deploy --dry-run --expect 0123456789aZ
[ "$rc" = 2 ] && ok "a non-hex --expect is a usage error" || bad "non-hex expect: rc=$rc"
# ---- the lock -----------------------------------------------------------
setup lock
( exec 9>>"$C/lock"; PATH="$S:$PATH" flock -n 9 && echo "since then (pid 1, id 1-1)" >"$C/lock" && touch "$C/held" && /bin/sleep 3 ) &
while [ ! -f "$C/held" ]; do /bin/sleep 0.1; done
deploy --dry-run
wait
[ "$rc" = 1 ] && grep -q "another deploy is running — since then" "$C/out" && ok "a second deploy is refused, saying since when" || bad "lock: rc=$rc $(cat "$C/out")"
# ---- --require-green ----------------------------------------------------
setup red; echo '{"required":["quarry","race"],"unmet":["race"]}' >"$C/checkin.json"; deploy --dry-run --require-green
[ "$rc" = 1 ] && journal 'FAILED stage=green .*race' && ok "--require-green refuses a red required check" || bad "red: rc=$rc"
setup green; echo '{"required":["quarry","race"],"unmet":[]}' >"$C/checkin.json"; deploy --dry-run --require-green
[ "$rc" = 0 ] && ok "--require-green passes when every required check is green" || bad "green: rc=$rc $(tail -1 "$C/out")"
setup noreq; echo '{"required":[],"unmet":[]}' >"$C/checkin.json"; deploy --dry-run --require-green
[ "$rc" = 1 ] && journal 'FAILED stage=green .*requires no checks' && ok "--require-green refuses a trunk that requires nothing" || bad "noreq: rc=$rc"
setup oldforge; echo '{"checks":[]}' >"$C/checkin.json"; deploy --dry-run --require-green
[ "$rc" = 1 ] && journal 'FAILED stage=green' && ok "--require-green refuses a forge that does not say" || bad "oldforge: rc=$rc"
# ---- the build and the runner set ---------------------------------------
setup died; echo none >"$C/build"; deploy
[ "$rc" = 1 ] && journal 'FAILED stage=build .*died part-way' && grep -q "echo old" "$C/bin/quarry" \
&& ok "a build that dies before the runner downloads fails, installing nothing" || bad "died: rc=$rc $(tail -1 "$C/out")"
setup pre; echo none >"$C/build"; grep -v '^RUNNER_TARGETS=' "$here/build-linux.sh" >"$C/tree/scripts/build-linux.sh"; deploy
[ "$rc" = 0 ] && journal '^OK ' && [ "$(cat "$C/bin/runner/SHA256SUMS")" = old ] && ok "a tree that builds no runner downloads deploys and keeps the published set" || bad "pre: rc=$rc $(tail -1 "$C/out")"
setup partial; echo partial >"$C/build"; deploy
[ "$rc" = 1 ] && journal 'FAILED stage=runner ' && [ "$(cat "$C/bin/runner/SHA256SUMS")" = old ] && grep -q "echo quarry" "$C/bin/quarry" \
&& ok "an incomplete runner set is not published; the forge deploys; the deploy ends FAILED stage=runner" || bad "partial: rc=$rc $(tail -1 "$C/out")"
setup partialdry; echo partial >"$C/build"; deploy --dry-run
[ "$rc" = 1 ] && journal 'FAILED stage=build .*runner' && ok "a dry run fails on an incomplete runner set" || bad "partialdry: rc=$rc"
setup full; deploy
[ "$rc" = 0 ] && journal '^OK ' && [ -f "$C/bin/runner/quarry-runner-linux-arm64" ] && grep -q "on-active=5" "$C/systemd-run" \
&& ok "a complete deploy publishes the set, schedules the runner restart and logs OK" || bad "full: rc=$rc $(tail -1 "$C/out")"
setup restart; echo 1 >"$C/systemd-run.rc"; deploy
[ "$rc" = 1 ] && journal 'FAILED stage=runner-restart ' && ! journal '^OK ' && ok "a runner restart that cannot be scheduled ends FAILED stage=runner-restart" || bad "restart: rc=$rc $(tail -1 "$C/out")"
# ---- the real zit --------------------------------------------------------
# Every zit call deploy.sh makes — pull from an upstream, open trunk from
# the store, info current — run against the binary itself in scratch
# stores, and the stub's fixtures held to what it prints. A zit whose
# output drifts fails here rather than on the host.
if ! command -v zit >/dev/null 2>&1; then
echo "skip: no zit on PATH — the real-zit cases did not run"
else
setup real
# A COPY: the deploy installs the built zit over \$BIN/zit.
cp "$(command -v zit)" "$C/bin/zit"
(
mkdir "$C/up" && cd "$C/up" && zit init -q && cp -R "$C/tree/." . &&
zit add go.mod VERSION scripts/build-linux.sh scripts/runner-check.sh scripts/deploy.sh scripts/version-stamp.sh >/dev/null &&
zit commit -m one --user t >/dev/null 2>&1
) || bad "real zit: could not build the scratch upstream"
first=$(cd "$C/up" && zit info current | head -1)
cp -R "$C/up" "$C/store"
( cd "$C/up" && echo two >>go.mod && zit commit -m two --user t >/dev/null 2>&1 )
second=$(cd "$C/up" && zit info current | head -1)
# The stub's fixtures, against the real output.
line1=$(cd "$C/up" && zit info current | sed -n 1p)
line2=$(cd "$C/up" && zit info current | sed -n 2p)
bare1=$(cd "$C/up" && zit info | sed -n 1p)
opened=$(mkdir "$C/o" && cd "$C/o" && zit open trunk --store="$C/up/.zit")
case "$line1" in *[!0-9a-f]* | "") bad "real zit: info current line 1 is not a hash: $line1" ;; *)
[ "${#line1}" = 64 ] && [ "$line2" = "branch: trunk" ] && [ "$opened" = "opened $second on trunk" ] &&
case "$bare1" in "repository: "*) true ;; *) false ;; esac &&
ok "the zit stub's fixtures match the real zit (info current, bare info, open)" ||
bad "real zit output drifted from the stub: info current=[$line1 / $line2] info=[$bare1] open=[$opened]" ;;
esac
# An upstream to pull from: zit pull speaks only HTTP. The server
# carries its own deadline — an alarm set before exec survives it — so
# a harness killed part-way never leaves it serving.
port=$((20000 + $$ % 20000))
( cd "$C/up" && exec perl -e 'alarm 120; exec @ARGV or die' zit serve --port "$port" ) >"$C/serve.log" 2>&1 &
serve=$!; pids="$pids $serve"
n=0; until curl -s -o /dev/null "http://127.0.0.1:$port/" || [ "$n" -ge 50 ]; do n=$((n + 1)); /bin/sleep 0.1; done
export QUARRY_UPSTREAM="http://127.0.0.1:$port"
deploy --expect "$second"
unset QUARRY_UPSTREAM
[ "$rc" = 0 ] && journal '^OK ' && [ "$(cd "$C/store" && zit info trunk | head -1)" = "$second" ] &&
ok "real zit: pull from an upstream, open trunk, and a full --expect of the pulled tip deploy" ||
bad "real zit deploy: rc=$rc $(tail -2 "$C/out" | tr '\n' ' ')"
: >"$C/journal"
# The deploy above installed its build's zit (a stub) over $BIN/zit.
cp "$(command -v zit)" "$C/bin/zit"
deploy --dry-run --expect "$(echo "$second" | cut -c1-12)"
[ "$rc" = 0 ] && ok "real zit: a 12-character --expect of trunk's tip deploys" || bad "real zit 12-char: rc=$rc $(tail -1 "$C/out")"
deploy --dry-run --expect "$first"
[ "$rc" = 1 ] && journal 'FAILED stage=open .*trunk moved' && ok "real zit: a full --expect of an older check-in is refused" ||
bad "real zit moved: rc=$rc $(tail -1 "$C/out")"
kill "$serve" 2>/dev/null && wait "$serve" 2>/dev/null
fi
# ---- the version stamp --------------------------------------------------
# version-stamp.sh is the rule; the deploy passes the tree's answer to the
# build as VERSION.
V=$T/VERSION; echo 0.2.0 >"$V"
stamp() { sh "$here/version-stamp.sh" "$@" 2>/dev/null; }
[ "$(stamp "$V" "$H" release version-0.2.0)" = 0.2.0 ] && ok "a check-in tagged version-X.Y.Z stamps X.Y.Z" || bad "tagged stamp: $(stamp "$V" "$H" release version-0.2.0)"
[ "$(stamp "$V" "$H" version-0.1.0 release)" = "0.2.0-dev+$(echo "$H" | cut -c1-12)" ] && ok "another version's tag stamps X.Y.Z-dev+" || bad "other tag stamp: $(stamp "$V" "$H" version-0.1.0)"
[ "$(stamp "$V" "$(echo "$H" | cut -c1-12)")" = "0.2.0-dev+$(echo "$H" | cut -c1-12)" ] && ok "an untagged check-in stamps X.Y.Z-dev+" || bad "untagged stamp"
[ "$(stamp "$V" local)" = "0.2.0-dev+local" ] && ok "a tree with no checkout stamps X.Y.Z-dev+local" || bad "local stamp"
echo 0.2 >"$T/VERSION.bad"
! stamp "$T/VERSION.bad" "$H" >/dev/null && ! stamp "$V" 0123 >/dev/null && ok "a VERSION that is not X.Y.Z, or a short check-in, is refused" || bad "bad stamp inputs accepted"
setup st_rel; printf 'release\nversion-0.2.0\n' >"$C/tags"; deploy --dry-run
[ "$rc" = 0 ] && [ "$(cat "$C/stamp")" = 0.2.0 ] && ok "a deploy of a tip tagged version-0.2.0 builds 0.2.0" || bad "deploy stamp tagged: rc=$rc stamp=$(cat "$C/stamp" 2>/dev/null)"
setup st_dev; deploy --dry-run
[ "$rc" = 0 ] && [ "$(cat "$C/stamp")" = "0.2.0-dev+$(echo "$H" | cut -c1-12)" ] && ok "a deploy of an untagged tip builds X.Y.Z-dev+" || bad "deploy stamp dev: rc=$rc stamp=$(cat "$C/stamp" 2>/dev/null)"
setup st_old; rm "$C/tree/VERSION"; deploy --dry-run
[ "$rc" = 0 ] && [ "$(cat "$C/stamp")" = "0.1.0-beta+$(echo "$H" | cut -c1-12)" ] && ok "a tree from before VERSION keeps its old stamp" || bad "deploy stamp old: rc=$rc stamp=$(cat "$C/stamp" 2>/dev/null)"
# ---- --release and --rollback ------------------------------------------
# A release as RFC 0020 lays it out: per arch a tarball holding quarry and
# zit in one top directory, the five runner downloads, one SHA256SUMS.
mkrelease() {
v=$1; R=$C/rel/$v; mkdir -p "$R"
for arch in amd64 arm64; do
d=$C/pkg/quarry-$v-linux-$arch; mkdir -p "$d"
printf '#!/bin/sh\necho "quarry %s"\n' "${2:-$v}" >"$d/quarry"
printf '#!/bin/sh\necho "zit from %s"\n' "$v" >"$d/zit"
chmod +x "$d/quarry" "$d/zit"; echo log >"$d/CHANGELOG.md"; echo l >"$d/LICENSE"
tar -czf "$R/quarry-$v-linux-$arch.tar.gz" -C "$C/pkg" "quarry-$v-linux-$arch"
done
for f in linux-amd64 linux-arm64 darwin-arm64 darwin-amd64 windows-amd64.exe; do echo "runner $v $f" >"$R/quarry-runner-$f"; done
( cd "$R" && sha256sum ./* | sed 's| \./| |' >SHA256SUMS )
}
installed_old() { grep -q "echo old" "$C/bin/quarry" && [ "$(cat "$C/bin/runner/SHA256SUMS")" = old ] && [ ! -f "$C/systemctl" ]; }
for flags in "--release 0.2.0 --require-green" "--release 0.2.0 --expect $H" "--release 0.2" "--release" "--rollback --release 0.2.0" "--rollback --dry-run"; do
setup rflags
# shellcheck disable=SC2086
deploy $flags
[ "$rc" = 2 ] && ok "deploy.sh $flags is a usage error" || bad "flags $flags: rc=$rc"
done
setup rel_bad; mkrelease 0.2.0; echo tampered >>"$C/rel/0.2.0/quarry-runner-darwin-arm64"; deploy --release 0.2.0
[ "$rc" = 1 ] && journal 'FAILED stage=verify .*SHA256SUMS' && installed_old \
&& ok "a release whose sums do not verify stops before installing anything" || bad "rel_bad: rc=$rc $(tail -1 "$C/out")"
setup rel_none; deploy --release 0.9.0
[ "$rc" = 1 ] && journal 'FAILED stage=fetch ' && installed_old && ok "a release the forge does not have is refused" || bad "rel_none: rc=$rc"
setup rel_name; mkrelease 0.2.0; echo "$H ../escape" >>"$C/rel/0.2.0/SHA256SUMS"; deploy --release 0.2.0
[ "$rc" = 1 ] && journal 'FAILED stage=fetch .*file name' && installed_old && ok "a SHA256SUMS naming a path outside the release is refused" || bad "rel_name: rc=$rc $(tail -1 "$C/out")"
setup rel_wrong; mkrelease 0.2.0 0.1.9; deploy --release 0.2.0
[ "$rc" = 1 ] && journal 'FAILED stage=verify .*not quarry 0.2.0' && installed_old && ok "a release whose quarry names another version is refused" || bad "rel_wrong: rc=$rc $(tail -1 "$C/out")"
setup rel_dry; mkrelease 0.2.0; deploy --dry-run --release 0.2.0
[ "$rc" = 0 ] && installed_old && grep -q 'releases/0.2.0/quarry-runner-windows-amd64.exe' "$C/fetched" \
&& ok "a dry-run release fetches and verifies, installing nothing" || bad "rel_dry: rc=$rc $(tail -1 "$C/out")"
setup rel; mkrelease 0.2.0; deploy --release 0.2.0
n=$(grep -c 'quarry-runner-' "$C/bin/runner/SHA256SUMS" 2>/dev/null)
if [ "$rc" = 0 ] && journal '^OK .*tip=0.2.0' && [ "$("$C/bin/quarry" version)" = "quarry 0.2.0" ] && grep -q "echo old" "$C/bin/quarry.prev" \
&& [ "$("$C/bin/zit")" = "zit from 0.2.0" ] && grep -q "zit stub" "$C/bin/zit.prev" && [ "$n" = 5 ] && [ "$(find "$C/bin/runner" -name "quarry-runner-*" | wc -l | tr -d " ")" = 5 ] \
&& (cd "$C/bin/runner" && sha256sum -c --quiet SHA256SUMS >/dev/null 2>&1) && [ "$(cat "$C/bin/runner.prev/SHA256SUMS")" = old ] \
&& [ ! -f "$C/stamp" ] && grep -q "restart quarry.service" "$C/systemctl" && grep -q "on-active=5" "$C/systemd-run"; then
ok "a good release installs quarry, its zit and the whole runner set, keeps quarry.prev and zit.prev, builds nothing, restarts the runner last"
else
bad "rel: rc=$rc runners=$n $(tail -1 "$C/out")"
fi
: >"$C/journal"; rm -f "$C/systemctl"; deploy --rollback
[ "$rc" = 0 ] && journal '^OK ' && grep -q "echo old" "$C/bin/quarry" && [ ! -e "$C/bin/quarry.prev" ] \
&& grep -q "zit stub" "$C/bin/zit" && [ ! -e "$C/bin/zit.prev" ] \
&& [ "$(cat "$C/bin/runner/SHA256SUMS")" = old ] && grep -q "restart quarry.service" "$C/systemctl" \
&& ok "--rollback restores quarry.prev, zit.prev and the previous runner set, and restarts" || bad "rollback: rc=$rc $(tail -1 "$C/out")"
: >"$C/journal"; deploy --rollback
[ "$rc" = 1 ] && journal 'FAILED stage=reinstate .*no .*quarry.prev' && grep -q "echo old" "$C/bin/quarry" \
&& ok "--rollback with no quarry.prev changes nothing and says so" || bad "rollback twice: rc=$rc $(tail -1 "$C/out")"
setup rel_silent; mkrelease 0.2.0; touch "$C/down"; deploy --release 0.2.0
[ "$rc" = 1 ] && journal 'FAILED stage=health .*rolled back' && grep -q "echo old" "$C/bin/quarry" && grep -q "zit stub" "$C/bin/zit" \
&& ok "a release that does not answer is rolled back, its zit with it" || bad "rel_silent: rc=$rc $(tail -1 "$C/out")"
# ---- quarry-alert.sh ----------------------------------------------------
C=$T/alert; mkdir -p "$C"
stub curl 'printf "%s\n" "$@" >"$C/argv"; cat >"$C/stdin"; exit "$(cat "$C/curl.rc" 2>/dev/null || echo 0)"'
stub systemctl 'case "$*" in *Result*) echo exit-code ;; *InvocationID*) echo inv1 ;; esac'
stub journalctl 'echo "FAILED stage=ready tip=abc id=1-2 via=unit: not ready"'
alert() { PATH="$S:$PATH" QUARRY_ALERT_RUN="$C/run" QUARRY_NOTIFY_KEY=sekrit-key-123 sh "$here/quarry-alert.sh" "$@" >/dev/null 2>&1; rc=$?; }
alert --key k1 --message "hello"
if [ "$rc" = 0 ] && ! grep -q sekrit "$C/argv" && grep -q sekrit "$C/stdin"; then
ok "the notify key reaches curl on stdin, never on its argv"
else
bad "alert argv: rc=$rc argv=$(tr '\n' ' ' <"$C/argv")"
fi
echo 7 >"$C/curl.rc"; alert quarry
[ "$rc" = 1 ] && [ ! -s "$C/run/quarry.service" ] && ok "a failed send writes no quiet mark" || bad "mark after failed send: rc=$rc"
echo 0 >"$C/curl.rc"; : >"$C/argv"; alert quarry
[ "$rc" = 0 ] && [ -s "$C/argv" ] && [ -s "$C/run/quarry.service" ] && ok "the next failure sends, and marks" || bad "send after failed: rc=$rc"
: >"$C/argv"; alert quarry
[ "$rc" = 0 ] && [ ! -s "$C/argv" ] && ok "a failure inside the quiet period is not sent" || bad "quiet: rc=$rc"
echo garbage >"$C/run/quarry.service"; alert quarry
[ "$rc" = 0 ] && [ -s "$C/argv" ] && ok "a corrupt quiet mark reads as absent" || bad "corrupt mark: rc=$rc"
: >"$C/run/quarry.service"; : >"$C/argv"; alert quarry
[ "$rc" = 0 ] && [ -s "$C/argv" ] && ok "an empty quiet mark reads as absent" || bad "empty mark: rc=$rc"
# ---- quarry-deploy-watch.sh ---------------------------------------------
# Who holds the lock is read from a /proc/locks laid out as the kernel
# prints it (a FLOCK line keyed MAJOR:MINOR:INODE in hex), so this runs
# on macOS too; `stat -c '%d %i'` is stubbed with the lock file's real
# inode on device 2049 (08:01), which is what that line names.
C=$T/watch; mkdir -p "$C"
stub journalctl 'cat "$C/journal" 2>/dev/null'
stub stat '[ "$1 $2 $3" = "-L -c %d %i" ] || exit 1; echo "2049 $(ls -di "$4" | awk "{print \$1}")"'
printf '#!/bin/sh\necho "$2" >>"%s/alerts"\n' "$C" >"$C/alert"; chmod +x "$C/alert"
watch() { PATH="$S:$PATH" QUARRY_DEPLOY_LOCK="$C/lock" QUARRY_DEPLOY_STATE="$C/state" QUARRY_DEPLOY_WATCH="$C/w" QUARRY_ALERT="$C/alert" QUARRY_PROC_LOCKS="$C/locks" sh "$here/quarry-deploy-watch.sh"; }
old=$(( $(date +%s) - 900 ))
echo "since then (pid 4242, id $old-4242)" >"$C/lock"
ino=$(ls -di "$C/lock" | awk '{print $1}')
held_line="7: FLOCK ADVISORY WRITE 4242 08:01:$ino 0 EOF"
# Held by the kernel's account: a deploy past ten minutes, alerted once.
printf '%s\n' "1: OFDLCK ADVISORY READ -1 08:01:5045820 128 128" "$held_line" >"$C/locks"
watch; watch
[ "$(grep -c "deploy-long:$old-4242" "$C/alerts" 2>/dev/null)" = 1 ] && ok "a lock /proc/locks shows held past ten minutes alerts once" || bad "long: $(cat "$C/alerts" 2>/dev/null)"
# A stale note naming a LIVE process whose command line says deploy —
# a reused pid, or this very harness — is not a holder: only the kernel's
# lock table says who holds the lock. With the state mid-rollback, that
# is a deploy that died without a result.
sh -c '/bin/sleep 30' deploy.sh & live=$!
echo "since then (pid $live, id $old-$live)" >"$C/lock"
printf '%s\n' "1: OFDLCK ADVISORY READ -1 08:01:5045820 128 128" "2: FLOCK ADVISORY WRITE 1 08:01:999999999 0 EOF" >"$C/locks"
echo "$old-$live rollback abc" >"$C/state"; : >"$C/journal"; : >"$C/alerts"; watch; watch
kill "$live" 2>/dev/null; wait "$live" 2>/dev/null
if [ "$(grep -c "deploy-lost:$old-$live" "$C/alerts")" = 1 ] && ! grep -q deploy-long "$C/alerts"; then
ok "a live pid in a stale lock note is not a holder; the dead deploy alerts once"
else
bad "stale pid: $(cat "$C/alerts")"
fi
# A waiter holds nothing.
echo "$old-4243 ok abc" >"$C/state"; : >"$C/alerts"
echo "3: -> FLOCK ADVISORY WRITE 4243 08:01:$ino 0 EOF" >"$C/locks"; watch
[ ! -s "$C/alerts" ] && ok "a waiter's line in /proc/locks is not a holder" || bad "waiter: $(cat "$C/alerts")"
# No /proc/locks: no holder, never an alert about one.
rm -f "$C/locks"; : >"$C/alerts"; watch
[ ! -s "$C/alerts" ] && ok "without /proc/locks there is no holder" || bad "no locks: $(cat "$C/alerts")"
# The watcher never takes the lock to look: a deploy that starts while
# it looks would be refused.
stub flock 'echo "flock $*" >>"$C/flocked"; exit 1'
printf '%s\n' "$held_line" >"$C/locks"; watch
[ ! -f "$C/flocked" ] && ok "the watcher never takes the deploy lock" || bad "watcher called flock: $(cat "$C/flocked")"
if [ "$fails" -ne 0 ]; then echo "$fails FAILED"; exit 1; fi
echo "all ok"