#!/bin/sh
# Deploy trunk to zeitforge.app (ticket 8eb0ca531a).
#
#   0. take the deploy lock: two deploys never overlap
#   1. (optionally) pull the store from another forge — this one is the origin
#   2. open trunk into a scratch tree; with --require-green, refuse unless
#      trunk's required checks are green on that exact check-in
#   3. build there, in a golang container
#   4. refuse below a free-space floor (JetStream will not start on a full disk)
#   5. install beside the running binary, keep the previous one, restart;
#      the runner downloads only as a complete, verified set
#   6. health-check; if the new binary does not answer, put the old one back
#   7. restart the CI runner last, delayed, so a deploy it ran can report
#
#   deploy.sh [--dry-run] [--require-green] [--expect HASH]
#
#   --dry-run        build and verify only: install nothing, restart nothing
#   --require-green  refuse unless every check trunk requires is green on
#                    the check-in about to be deployed (asked of the forge)
#   --expect HASH    refuse unless trunk's tip is HASH (a prefix will do):
#                    a CI deploy ships the check-in it tested, never a newer one
#
# Every way this ends badly ends the same way: exit non-zero, one line on
# stderr naming the stage, and the same line in the journal under
# `logger -t quarry-deploy` as "FAILED stage=<stage> ...". A deploy run as
# quarry-deploy@<hash>.service is alerted on by its OnFailure=; a deploy
# run by hand is picked up from the journal by quarry-deploy-watch.timer
# (scripts/systemd, docs/running.md).
set -eu
STORE=/srv/quarry/repos/quarry
BIN=/srv/quarry/bin
ZITREL=/home/dmestas/zit-release
FLOOR_KB=$((3*1024*1024))
# Overridable so the lock and the state can be exercised off the host.
LOCK=${QUARRY_DEPLOY_LOCK:-/run/quarry-deploy.lock}
STATE=${QUARRY_DEPLOY_STATE:-/var/lib/quarry-deploy/state}
API=${QUARRY_DEPLOY_API:-http://127.0.0.1:8950}

dry=; green=; expect=
while [ $# -gt 0 ]; do
  case "$1" in
    --dry-run) dry=1 ;;
    --require-green) green=1 ;;
    --expect) [ $# -ge 2 ] || { echo "deploy: --expect needs a hash" >&2; exit 2; }; expect=$2; shift ;;
    *) echo "usage: deploy.sh [--dry-run] [--require-green] [--expect HASH]" >&2; exit 2 ;;
  esac
  shift
done

# stage is what the deploy is doing now; the exit trap names it when the
# script dies somewhere nobody wrote a message for (set -e). The state
# file carries it too, for the watcher: a state that is neither ok nor
# failed, with the lock free, is a deploy that died without saying so.
id="$(date +%s)-$$"
stage=start
tip=
# quarry-deploy@.service sets QUARRY_DEPLOY_VIA=unit: its OnFailure=
# alerts for it, and the watcher alerts only for the rest.
via=${QUARRY_DEPLOY_VIA:-manual}
said=
work=
say_state() {
  [ -n "$dry" ] && return 0
  mkdir -p "$(dirname "$STATE")" 2>/dev/null || true
  printf '%s %s %s\n' "$id" "$1" "${tip:-unknown}" >"$STATE" 2>/dev/null || true
}
at() { stage=$1; say_state "$1"; }
fail() {
  said=1
  echo "deploy: FAILED at $stage: $1" >&2
  logger -t quarry-deploy "FAILED stage=$stage tip=${tip:-unknown} id=$id via=$via${dry:+ dry-run}: $1" 2>/dev/null || true
  say_state failed
  exit 1
}
on_exit() {
  rc=$?
  [ -n "$work" ] && rm -rf "$work"
  if [ "$rc" -ne 0 ] && [ -z "$said" ]; then
    said=1
    echo "deploy: FAILED at $stage: exited $rc" >&2
    logger -t quarry-deploy "FAILED stage=$stage tip=${tip:-unknown} id=$id via=$via${dry:+ dry-run}: exited $rc" 2>/dev/null || true
    say_state failed
  fi
}
trap on_exit EXIT
trap 'exit 130' INT TERM HUP

# The lock. flock on fd 9, opened for APPEND so a second deploy does not
# truncate the first one's note of when it started; only the holder
# rewrites it. /run is tmpfs, so a reboot is the one thing that clears
# it, and the kernel drops the lock with the last descriptor — a killed
# deploy never leaves it held.
stage=lock
exec 9>>"$LOCK"
if ! flock -n 9; then
  said=1
  echo "deploy: another deploy is running — $(cat "$LOCK" 2>/dev/null || echo "since an unknown time")" >&2
  exit 1
fi
printf 'since %s (pid %s, id %s)\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$$" "$id" >"$LOCK"
at start
logger -t quarry-deploy "START id=$id via=$via${dry:+ dry-run}" 2>/dev/null || true

at space
free_kb() { df -k / | awk 'NR==2 {print $4}'; }
if [ "$(free_kb)" -lt "$FLOOR_KB" ]; then
  fail "$(( $(free_kb) / 1024 )) MB free, need $(( FLOOR_KB / 1024 )) MB"
fi
# This store is the ORIGIN: landings are accepted here and the merge
# is already in it. QUARRY_UPSTREAM is only for pulling from another
# forge on purpose (a migration); unset, nothing is pulled. A dry run
# changes nothing, the store included.
if [ -n "${QUARRY_UPSTREAM:-}" ] && [ -z "$dry" ]; then
  at pull
  n=0
  until sudo -u quarry sh -c "cd $STORE && $BIN/zit pull $QUARRY_UPSTREAM 2>&1 | tail -1 | tee /dev/stderr | grep -qvE 'refused|failed|error|cannot'"; do
    n=$((n+1)); [ "$n" -lt 8 ] || fail "pull from $QUARRY_UPSTREAM kept failing"
    sleep 8
  done
fi
at open
work=$(mktemp -d /tmp/quarry-deploy.XXXXXX)
chown quarry:quarry "$work"
sudo -u quarry sh -c "cd $work && $BIN/zit open trunk --store=$STORE/.zit >/dev/null"
full=$(cd "$work" && sudo -u quarry $BIN/zit status | awk '/^checkout:/ {print $2}')
tip=$(printf '%s' "$full" | cut -c1-12)
[ -n "$tip" ] || fail "no checkout hash in the scratch tree"
say_state open
if [ -n "$expect" ]; then
  case "$full" in
    "$expect"*) ;;
    *) fail "trunk is at $tip, not $expect — trunk moved since it was tested" ;;
  esac
fi
# --require-green asks the forge, not the tree: which checks trunk
# requires is policy the forge keeps (RFC 0009), and whether each is
# green on this exact check-in is the forge's answer too, by the rule a
# landing's accept uses (checkin.get's required and unmet). Nothing is
# hard-coded here. A trunk that requires nothing is refused: "green" of
# nothing is not a reason to ship.
if [ -n "$green" ]; then
  at green
  js=$(curl -sf -m 10 "$API/api/v1/r/$(basename "$STORE")/checkin/$full") || fail "the forge did not answer checkin.get for $tip"
  req=$(printf '%s' "$js" | jq -r 'if (.required|type) == "array" then .required|join(" ") else "?" end')
  unmet=$(printf '%s' "$js" | jq -r 'if (.unmet|type) == "array" then .unmet|join(" ") else "?" end')
  [ "$req" != "?" ] || fail "the forge's checkin.get names no required checks (a forge older than this script)"
  [ -n "$req" ] || fail "trunk requires no checks; --require-green has nothing to require"
  [ -z "$unmet" ] || fail "required checks not green on $tip: $unmet (required: $req)"
  echo "deploy: $tip is green for trunk ($req)"
fi
# This script is BOTH tracked here and installed at /srv/quarry/deploy.sh,
# and the installed copy is the one that runs. So the two can drift, and
# on 2026-09-19 they did: the installed copy was edited by hand to build
# and install the CI worker, with no review possible because the file was
# not in the repository. That is the same silence that let the worker go
# fifteen days and one protocol migration without being rebuilt.
#
# Warn, never fail. A deploy that refuses to run because its own script
# disagrees with the tree is worse than one that tells you they disagree
# — and this script must keep working when the tree's copy is the broken
# one, which is exactly when you need to deploy.
if [ -f "$work/scripts/deploy.sh" ] && ! cmp -s "$0" "$work/scripts/deploy.sh"; then
  echo "deploy: WARNING — $0 differs from scripts/deploy.sh in trunk $tip" >&2
  echo "deploy:   diff -u $0 $work/scripts/deploy.sh" >&2
  echo "deploy:   install -m 0755 $work/scripts/deploy.sh $0   # once the tree's copy is the right one" >&2
fi
# The build image is the Go that go.mod names — one pin, not two. And
# every deploy looks for a newer stable Go: quarry is meant to track the
# latest (e3dfc93f), and a warning here is the one place an operator
# reads on every release. Warn, never fail, like the check above.
gover=$(sed -n 's/^go //p' "$work/go.mod")
latest=$(curl -s -m 10 'https://go.dev/dl/?mode=json' | grep -o '"version": *"go[0-9.]*"' | head -1 | grep -o '[0-9][0-9.]*') || true
if [ -n "$latest" ] && [ "$latest" != "$gover" ]; then
  echo "deploy: WARNING — go.mod builds with Go $gover; Go $latest is the latest stable. Bump it: go mod edit -go=$latest && go mod tidy" >&2
fi
at build
echo "deploy: building trunk $tip with Go $gover"
docker run --rm -v "$work":/src -v "$ZITREL":/zitrel:ro -w /src -e GOTOOLCHAIN=auto -e ZIT_RELEASE_DIR=/zitrel \
  -e VERSION="0.1.0-beta+$tip" -e PREFIX=/src/bin "golang:$gover" sh -c "scripts/build-linux.sh" 2>&1 | grep -v "^go: downloading" | tail -2
[ -x "$work/bin/quarry" ] || fail "build produced no binary"
[ -x "$work/bin/zit" ] || fail "build produced no zit"
"$work/bin/quarry" version >/dev/null 2>&1 || fail "the built binary does not run"
# The runner downloads are published all or nothing. A cross-build that
# fails for one platform stops build-linux.sh part-way — some binaries,
# no SHA256SUMS — and the docker pipe above hides its exit status, so a
# partial set used to replace the whole published one (2026-09-30: an
# arm64-only compile error dropped SHA256SUMS and four downloads). The
# check is the tree's own (scripts/runner-check.sh reads build-linux.sh's
# platform list). A bad set is not installed, the forge still deploys,
# and the deploy ENDS FAILED at stage runner once everything else is
# done: the forge is fine, the downloads are stale, and somebody has to
# hear it.
runner_bad=
if [ -d "$work/bin/runner" ]; then
  if [ -f "$work/scripts/runner-check.sh" ]; then
    runner_bad=$(sh "$work/scripts/runner-check.sh" "$work/bin/runner" "$work/scripts/build-linux.sh") || true
    if [ -n "$runner_bad" ]; then
      echo "deploy: runner downloads incomplete, keeping the published set: $runner_bad" >&2
    fi
  else
    runner_bad="trunk has no scripts/runner-check.sh to check the set with"
  fi
fi
if [ -n "$dry" ]; then
  [ -z "$runner_bad" ] || fail "runner downloads incomplete: $runner_bad"
  logger -t quarry-deploy "OK id=$id via=$via dry-run tip=$tip" 2>/dev/null || true
  said=1
  echo "deploy: dry run — trunk $tip builds and runs; nothing installed, nothing restarted"
  exit 0
fi
at install
cp -p "$BIN/quarry" "$BIN/quarry.prev" 2>/dev/null || true
install -m 0755 "$work/bin/quarry" "$BIN/quarry.new" && mv -f "$BIN/quarry.new" "$BIN/quarry"
install -m 0755 "$work/bin/zit" "$BIN/zit"
# The quarry-runner downloads (served at /runner/ from $BIN/runner, the
# directory beside the binary). Swapped whole, so a visitor never gets
# one platform's new build beside another's old one; a build that
# predates them, or whose set is incomplete, leaves the installed set alone.
if [ -d "$work/bin/runner" ] && [ -z "$runner_bad" ]; then
  rm -rf "$BIN/runner.new" && cp -r "$work/bin/runner" "$BIN/runner.new" && chmod 0755 "$BIN/runner.new"/quarry-runner-*
  rm -rf "$BIN/runner.prev"; [ -d "$BIN/runner" ] && mv "$BIN/runner" "$BIN/runner.prev"
  mv "$BIN/runner.new" "$BIN/runner"
fi
at restart
systemctl restart quarry.service
answers() {
  for _ in $(seq 1 30); do
    if curl -sf -m 3 -o /dev/null http://127.0.0.1:8950/; then return 0; fi
    sleep 2
  done
  return 1
}
at health
if ! answers; then
  echo "deploy: new binary did not answer in 60s - rolling back" >&2
  at rollback
  if ! { mv -f "$BIN/quarry.prev" "$BIN/quarry" && systemctl restart quarry.service; }; then
    fail "could not put the previous binary back; quarry may be down"
  fi
  answers || fail "put the previous binary back and it does not answer either; quarry is down"
  stage=health
  fail "new binary did not answer in 60s; rolled back to the previous one"
fi
# The port answering is not the forge answering. After a restart it
# registers its repositories one by one — 65-75s for two dozen — and
# refuses every sync and every repository page as "still starting" for
# that whole window, while / answers 200. This script used to print
# success at the top of that window, so the operator's last signal was
# "serving" while the forge was down for another minute (b81ae623).
# So: wait for ready, say how long it took, and refuse to call a forge
# that never gets there healthy.
at ready
started=$(date +%s)
ready=
for _ in $(seq 1 150); do
  if curl -sf -m 3 http://127.0.0.1:8950/api/v1/version | grep -q '"ready":true'; then ready=1; break; fi
  sleep 2
done
waited=$(( $(date +%s) - started ))
if [ "$ready" != 1 ]; then
  echo "deploy: NOT rolling back — the binary answers; registration is what is slow. Watch: journalctl -u quarry -f" >&2
  fail "forge is up but not ready after ${waited}s; still starting: $(curl -s -m 3 http://127.0.0.1:8950/api/v1/version | grep -o '"starting":\[[^]]*\]')"
fi
echo "deploy: zeitforge.app serving $(curl -s http://127.0.0.1:8950/ | grep -o 'quarry 0[^<]*' | head -1 | sed 's/&#43;/+/') — ready after ${waited}s"
# 8. warm the edge: every deploy renames the hashed assets, so the first
#    visitor at each Cloudflare location would otherwise pay the miss.
#    Best-effort: a cold edge is slow, not broken.
at warm
warm() { curl -s -m 10 -o /dev/null -H "Accept-Encoding: gzip" "https://zeitforge.app$1" || true; }
warm / ; warm /start
for a in $(curl -s -m 10 https://zeitforge.app/ | grep -o "/assets/[^\"]*" | sort -u); do warm "$a"; done
echo "deploy: edge warmed"
if [ -z "$runner_bad" ]; then
  said=1
  logger -t quarry-deploy "OK id=$id via=$via tip=$tip ready=${waited}s" 2>/dev/null || true
  say_state ok
fi
# quarry's own CI runs on `quarry runner` (quarry-runner.service, unit in
# /etc/systemd/system, token in /etc/quarry-runner.env). It is this same
# binary, so it restarts onto the new build — LAST, and five seconds from
# now rather than now: a deploy the runner itself started is a child of
# that runner, and restarting it inline would kill the deploy before it
# could report. systemd-run hands the restart to a transient timer that
# outlives this script. A host that never installed the runner unit
# skips this quietly.
if systemctl cat quarry-runner.service >/dev/null 2>&1; then
  systemd-run --quiet --on-active=5 /bin/systemctl restart quarry-runner.service \
    && echo "deploy: quarry runner restarts in 5s"
fi
if [ -n "$runner_bad" ]; then
  stage=runner
  fail "forge deployed and ready, but the runner downloads were NOT published (kept the previous set): $runner_bad"
fi