#!/bin/sh
# Tell the forge's admins that something on this host went wrong, through
# the notify companion quarry already uses: the quarry-ops workflow
# (inbox + push, internal/notify/workflows.go), addressed to the admins
# topic. Installed at /srv/quarry/quarry-alert.sh (docs/running.md).
#
# quarry-alert.sh UNIT a unit failed: OnFailure=quarry-alert@%N.service
# quarry-alert.sh --key KEY --message TEXT anything else (quarry-deploy-watch.sh)
#
# QUARRY_NOTIFY_KEY is notify's API key, the one quarry.service reads: the
# units run this under `doppler run --project quarry --config prd`, with
# the service token from /srv/quarry/env. Every alert is also a journal
# line (logger -t quarry-alert), so one that could not be sent is still
# on the host.
set -eu
NOTIFY=${QUARRY_NOTIFY_URL:-http://127.0.0.1:3000}
TOPIC=${QUARRY_ALERT_TOPIC:-admins}
ORIGIN=${QUARRY_ORIGIN:-https://zeitforge.app}
# A unit in a crash loop fails every few seconds (quarry.service has
# Restart=on-failure, and OnFailure= fires on each one): one alert per
# unit per QUIET seconds is news, the rest is noise.
QUIET=${QUARRY_ALERT_QUIET:-600}
RUN=${QUARRY_ALERT_RUN:-/run/quarry-alert}
key=; text=; unit=
case "${1:-}" in
--key)
[ $# -eq 4 ] && [ "$3" = --message ] || { echo "usage: quarry-alert.sh UNIT | --key KEY --message TEXT" >&2; exit 2; }
key=$2; text=$4; summary=$(printf '%s' "$text" | head -1 | cut -c1-120) ;;
"" | -*) echo "usage: quarry-alert.sh UNIT | --key KEY --message TEXT" >&2; exit 2 ;;
*)
unit=$1
case "$unit" in *.*) ;; *) unit=$unit.service ;; esac
mkdir -p "$RUN"
mark="$RUN/$unit"
now=$(date +%s)
# The mark is the time of the last alert SENT; anything but a
# number in it (empty, torn, corrupt) reads as no mark at all.
last=$(cat "$mark" 2>/dev/null || true)
case "$last" in
"" | *[!0-9]*) last= ;;
esac
if [ -n "$last" ] && [ $((now - last)) -lt "$QUIET" ]; then
logger -t quarry-alert "quiet: $unit failed again within ${QUIET}s; not sent"
exit 0
fi
result=$(systemctl show -p Result --value "$unit" 2>/dev/null || echo unknown)
inv=$(systemctl show -p InvocationID --value "$unit" 2>/dev/null || true)
# The failed run's own lines: a deploy's FAILED line when it wrote
# one, else the last few lines the unit logged.
if [ -n "$inv" ]; then
log=$(journalctl -o cat _SYSTEMD_INVOCATION_ID="$inv" 2>/dev/null || true)
else
log=$(journalctl -o cat -n 20 -u "$unit" 2>/dev/null || true)
fi
text=$(printf '%s\n' "$log" | grep 'FAILED stage=' | tail -1 || true)
[ -n "$text" ] || text=$(printf '%s\n' "$log" | tail -5)
summary="$unit failed on $(hostname) ($result)"
key="unit:$unit:${inv:-$now}" ;;
esac
logger -t quarry-alert "$summary — $(printf '%s' "$text" | tr '\n' ' ')"
if [ -z "${QUARRY_NOTIFY_KEY:-}" ]; then
logger -t quarry-alert "not sent: no QUARRY_NOTIFY_KEY (run under doppler run --project quarry --config prd)"
exit 1
fi
body=$(jq -n --arg topic "$TOPIC" --arg summary "$summary" --arg text "$text" \
--arg link "$ORIGIN/" --arg key "$key" \
'{name: "quarry-ops", to: {topicKey: $topic},
payload: {summary: $summary, text: $text, link: $link},
transactionId: $key}')
# The key goes to curl on stdin (-H @-), never on its command line:
# /proc is readable by every account on the host, CI jobs included.
if ! printf 'Authorization: ApiKey %s\n' "$QUARRY_NOTIFY_KEY" |
curl -sf -m 10 -o /dev/null -X POST "$NOTIFY/v1/events/trigger" \
-H 'Content-Type: application/json' -H @- --data-binary "$body"; then
logger -t quarry-alert "not sent: notify at $NOTIFY refused or did not answer"
exit 1
fi
# Quiet only after an alert actually went: one that failed to send must
# not silence the next.
[ -z "$unit" ] || echo "$now" >"$RUN/$unit"