#!/bin/sh
# Watch deploys for the failures that end with nobody told. Run every
# five minutes by quarry-deploy-watch.timer (docs/running.md); each
# finding alerts once, through quarry-alert.sh.
#
# 1. a deploy has held the lock for over ten minutes
# 2. a deploy ended — its lock is free — while its state says it was
# mid-way (installing, restarting, rolling back) and the journal has
# no result line for it: killed, or its terminal went away
# 3. a deploy run by hand wrote a FAILED line: a unit's failure is
# alerted by its own OnFailure=, a terminal's only by this
set -eu
LOCK=${QUARRY_DEPLOY_LOCK:-/run/quarry-deploy.lock}
STATE=${QUARRY_DEPLOY_STATE:-/var/lib/quarry-deploy/state}
WATCH=${QUARRY_DEPLOY_WATCH:-/var/lib/quarry-deploy-watch}
ALERT=${QUARRY_ALERT:-/srv/quarry/quarry-alert.sh}
LIMIT=${QUARRY_DEPLOY_LIMIT:-600}
LOCKS=${QUARRY_PROC_LOCKS:-/proc/locks}
mkdir -p "$WATCH"
now=$(date +%s)
# once KEY MESSAGE: alert unless KEY already was. An alert that could
# not be sent is tried again on the next run (no mark), and never stops
# this one: the other checks still run.
once() {
mark="$WATCH/sent.$(printf '%s' "$1" | tr -c 'A-Za-z0-9._-' '_')"
[ -f "$mark" ] && return 0
if "$ALERT" --key "$1" --message "$2"; then
touch "$mark"
else
logger -t quarry-deploy-watch "alert not sent: $1" || true
fi
}
# Is a deploy running? Ask the kernel, read-only: /proc/locks lists
# every flock held, by device and inode, and a WRITE flock on the lock
# file's inode is a deploy holding it — whatever pid wrote the note in
# the file, and whether or not that pid has since been reused. Taking
# the lock to test it would refuse a deploy starting that instant, and
# a pid or a command line matches the wrong process (this watcher's own
# "deploy" in its name among them). No /proc/locks, no holder.
held=
if [ -f "$LOCK" ] && [ -r "$LOCKS" ]; then
# %d is the file's device number (NOT %t/%T, which are a device
# node's own type); the kernel prints it as MAJOR:MINOR in hex.
# shellcheck disable=SC2046 # two words, split on purpose
set -- $(stat -L -c '%d %i' "$LOCK" 2>/dev/null || true)
if [ $# -eq 2 ]; then
want=$(printf '%02x:%02x:%s' $(( ($1 >> 8) & 0xfff )) $(( ($1 & 0xff) | (($1 >> 12) & 0xfff00) )) "$2")
# "1: FLOCK ADVISORY WRITE <pid> <maj>:<min>:<inode> 0 EOF"; a
# waiter's line reads "1: -> FLOCK ..." and holds nothing.
if awk -v want="$want" '$2 == "FLOCK" && $4 == "WRITE" && tolower($6) == want { found = 1 } END { exit !found }' "$LOCKS"; then
held=1
fi
fi
fi
# 1. Too long. The holder's id is "<start epoch>-<pid>".
if [ -n "$held" ]; then
id=$(sed -n 's/.*id \([0-9]*-[0-9]*\)).*/\1/p' "$LOCK")
start=${id%%-*}
if [ -n "$start" ] && [ $((now - start)) -gt "$LIMIT" ]; then
once "deploy-long:$id" "a deploy on $(hostname) has run $(( (now - start) / 60 )) min: $(cat "$LOCK")"
fi
fi
# 2. Ended without a result.
if [ -z "$held" ] && [ -f "$STATE" ]; then
read -r id stage tip <"$STATE" || true
case "$stage" in
ok | failed | "") ;;
*)
if ! journalctl -t quarry-deploy -o cat --since "@${id%%-*}" 2>/dev/null | grep -qE "^(OK|FAILED) .*id=$id( |$)"; then
what="ended during $stage"
[ "$stage" = rollback ] && what="began a rollback"
once "deploy-lost:$id" "a deploy of $tip on $(hostname) $what and recorded no result; check which binary is serving (journalctl -t quarry-deploy, /srv/quarry/bin)"
fi ;;
esac
fi
# 3. FAILED lines from deploys run by hand, since the last look. The
# cursor file makes each line read once; the first run looks back one
# interval rather than at all of history.
cur="$WATCH/cursor"
if [ -s "$cur" ]; then
lines=$(journalctl -t quarry-deploy -o cat --cursor-file="$cur" 2>/dev/null || true)
else
lines=$(journalctl -t quarry-deploy -o cat --since=-6min --cursor-file="$cur" 2>/dev/null || true)
fi
printf '%s\n' "$lines" | grep '^FAILED .* via=manual:' | while IFS= read -r line; do
id=$(printf '%s' "$line" | sed -n 's/.* id=\([0-9]*-[0-9]*\) .*/\1/p')
once "deploy-fail:${id:-$now}" "deploy on $(hostname): $line"
done