#!/bin/sh
# Watch deploys for the failures that end with nobody told. Run every
# five minutes by quarry-deploy-watch.timer (docs/running.md); each
# finding alerts once, through quarry-alert.sh.
#
# 1. a deploy has held the lock for over ten minutes
# 2. a deploy ended — its lock is free — while its state says it was
# mid-way (installing, restarting, rolling back) and the journal has
# no result line for it: killed, or its terminal went away
# 3. a deploy run by hand wrote a FAILED line: a unit's failure is
# alerted by its own OnFailure=, a terminal's only by this
set -eu
LOCK=${QUARRY_DEPLOY_LOCK:-/run/quarry-deploy.lock}
STATE=${QUARRY_DEPLOY_STATE:-/var/lib/quarry-deploy/state}
WATCH=${QUARRY_DEPLOY_WATCH:-/var/lib/quarry-deploy-watch}
ALERT=${QUARRY_ALERT:-/srv/quarry/quarry-alert.sh}
LIMIT=${QUARRY_DEPLOY_LIMIT:-600}
mkdir -p "$WATCH"
now=$(date +%s)
# once KEY MESSAGE: alert unless KEY already was. An alert that could
# not be sent is tried again on the next run (no mark), and never stops
# this one: the other checks still run.
once() {
mark="$WATCH/sent.$(printf '%s' "$1" | tr -c 'A-Za-z0-9._-' '_')"
[ -f "$mark" ] && return 0
if "$ALERT" --key "$1" --message "$2"; then
touch "$mark"
else
logger -t quarry-deploy-watch "alert not sent: $1" || true
fi
}
# Is a deploy running? Read, never lock: taking the lock to test it,
# even for an instant, refuses a deploy that starts in that instant. The
# holder writes "(pid N, id …)" into the lock file once it holds it; a
# live N — on Linux, one whose command line names deploy, so a reused
# pid is not mistaken for it — is a deploy still running.
held=
pid=$(sed -n 's/.*(pid \([0-9]*\),.*/\1/p' "$LOCK" 2>/dev/null || true)
if [ -n "$pid" ] && kill -0 "$pid" 2>/dev/null; then
if [ ! -r "/proc/$pid/cmdline" ] || tr '\0' ' ' <"/proc/$pid/cmdline" | grep -q deploy; then
held=1
fi
fi
# 1. Too long. The holder's id is "<start epoch>-<pid>".
if [ -n "$held" ]; then
id=$(sed -n 's/.*id \([0-9]*-[0-9]*\)).*/\1/p' "$LOCK")
start=${id%%-*}
if [ -n "$start" ] && [ $((now - start)) -gt "$LIMIT" ]; then
once "deploy-long:$id" "a deploy on $(hostname) has run $(( (now - start) / 60 )) min: $(cat "$LOCK")"
fi
fi
# 2. Ended without a result.
if [ -z "$held" ] && [ -f "$STATE" ]; then
read -r id stage tip <"$STATE" || true
case "$stage" in
ok | failed | "") ;;
*)
if ! journalctl -t quarry-deploy -o cat --since "@${id%%-*}" 2>/dev/null | grep -qE "^(OK|FAILED) .*id=$id( |$)"; then
what="ended during $stage"
[ "$stage" = rollback ] && what="began a rollback"
once "deploy-lost:$id" "a deploy of $tip on $(hostname) $what and recorded no result; check which binary is serving (journalctl -t quarry-deploy, /srv/quarry/bin)"
fi ;;
esac
fi
# 3. FAILED lines from deploys run by hand, since the last look. The
# cursor file makes each line read once; the first run looks back one
# interval rather than at all of history.
cur="$WATCH/cursor"
if [ -s "$cur" ]; then
lines=$(journalctl -t quarry-deploy -o cat --cursor-file="$cur" 2>/dev/null || true)
else
lines=$(journalctl -t quarry-deploy -o cat --since=-6min --cursor-file="$cur" 2>/dev/null || true)
fi
printf '%s\n' "$lines" | grep '^FAILED .* via=manual:' | while IFS= read -r line; do
id=$(printf '%s' "$line" | sed -n 's/.* id=\([0-9]*-[0-9]*\) .*/\1/p')
once "deploy-fail:${id:-$now}" "deploy on $(hostname): $line"
done