#!/bin/sh
# Tell the forge's admins that something on this host went wrong, through
# the notify companion quarry already uses: the quarry-ops workflow
# (inbox + push, internal/notify/workflows.go), addressed to the admins
# topic. Installed at /srv/quarry/quarry-alert.sh (docs/running.md).
#
#   quarry-alert.sh UNIT                       a unit failed: OnFailure=quarry-alert@%N.service
#   quarry-alert.sh --key KEY --message TEXT   anything else (quarry-deploy-watch.sh)
#
# QUARRY_NOTIFY_KEY is notify's API key, the one quarry.service reads: the
# units run this under `doppler run --project quarry --config prd`, with
# the service token from /srv/quarry/env. Every alert is also a journal
# line (logger -t quarry-alert), so one that could not be sent is still
# on the host.
set -eu
NOTIFY=${QUARRY_NOTIFY_URL:-http://127.0.0.1:3000}
TOPIC=${QUARRY_ALERT_TOPIC:-admins}
ORIGIN=${QUARRY_ORIGIN:-https://zeitforge.app}
# A unit in a crash loop fails every few seconds (quarry.service has
# Restart=on-failure, and OnFailure= fires on each one): one alert per
# unit per QUIET seconds is news, the rest is noise.
QUIET=${QUARRY_ALERT_QUIET:-600}
RUN=${QUARRY_ALERT_RUN:-/run/quarry-alert}

key=; text=; unit=
case "${1:-}" in
  --key)
    [ $# -eq 4 ] && [ "$3" = --message ] || { echo "usage: quarry-alert.sh UNIT | --key KEY --message TEXT" >&2; exit 2; }
    key=$2; text=$4; summary=$(printf '%s' "$text" | head -1 | cut -c1-120) ;;
  "" | -*) echo "usage: quarry-alert.sh UNIT | --key KEY --message TEXT" >&2; exit 2 ;;
  *)
    unit=$1
    case "$unit" in *.*) ;; *) unit=$unit.service ;; esac
    mkdir -p "$RUN"
    mark="$RUN/$unit"
    now=$(date +%s)
    if [ -f "$mark" ] && [ $((now - $(cat "$mark"))) -lt "$QUIET" ]; then
      logger -t quarry-alert "quiet: $unit failed again within ${QUIET}s; not sent"
      exit 0
    fi
    echo "$now" >"$mark"
    result=$(systemctl show -p Result --value "$unit" 2>/dev/null || echo unknown)
    inv=$(systemctl show -p InvocationID --value "$unit" 2>/dev/null || true)
    # The failed run's own lines: a deploy's FAILED line when it wrote
    # one, else the last few lines the unit logged.
    if [ -n "$inv" ]; then
      log=$(journalctl -o cat _SYSTEMD_INVOCATION_ID="$inv" 2>/dev/null || true)
    else
      log=$(journalctl -o cat -n 20 -u "$unit" 2>/dev/null || true)
    fi
    text=$(printf '%s\n' "$log" | grep 'FAILED stage=' | tail -1 || true)
    [ -n "$text" ] || text=$(printf '%s\n' "$log" | tail -5)
    summary="$unit failed on $(hostname) ($result)"
    key="unit:$unit:${inv:-$now}" ;;
esac

logger -t quarry-alert "$summary — $(printf '%s' "$text" | tr '\n' ' ')"
if [ -z "${QUARRY_NOTIFY_KEY:-}" ]; then
  logger -t quarry-alert "not sent: no QUARRY_NOTIFY_KEY (run under doppler run --project quarry --config prd)"
  exit 1
fi
body=$(jq -n --arg topic "$TOPIC" --arg summary "$summary" --arg text "$text" \
  --arg link "$ORIGIN/" --arg key "$key" \
  '{name: "quarry-ops", to: {topicKey: $topic},
    payload: {summary: $summary, text: $text, link: $link},
    transactionId: $key}')
if ! curl -sf -m 10 -o /dev/null -X POST "$NOTIFY/v1/events/trigger" \
  -H 'Content-Type: application/json' -H "Authorization: ApiKey $QUARRY_NOTIFY_KEY" \
  --data-binary "$body"; then
  logger -t quarry-alert "not sent: notify at $NOTIFY refused or did not answer"
  exit 1
fi