#!/bin/sh
# Deploy trunk to zeitforge.app (ticket 8eb0ca531a).
#
# 1. (optionally) pull the store from another forge — this one is the origin
# 2. open trunk into a scratch tree and build there, in a golang container
# 3. refuse below a free-space floor (JetStream will not start on a full disk)
# 4. install beside the running binary, keep the previous one, restart
# 5. health-check; if the new binary does not answer, put the old one back
set -eu
STORE=/srv/quarry/repos/quarry
BIN=/srv/quarry/bin
ZITREL=/home/dmestas/zit-release
FLOOR_KB=$((3*1024*1024))
free_kb() { df -k / | awk 'NR==2 {print $4}'; }
if [ "$(free_kb)" -lt "$FLOOR_KB" ]; then
echo "deploy: $(( $(free_kb) / 1024 )) MB free, need $(( FLOOR_KB / 1024 )) MB" >&2; exit 1
fi
# This store is the ORIGIN: landings are accepted here and the merge
# is already in it. QUARRY_UPSTREAM is only for pulling from another
# forge on purpose (a migration); unset, nothing is pulled.
if [ -n "${QUARRY_UPSTREAM:-}" ]; then
n=0
until sudo -u quarry sh -c "cd $STORE && $BIN/zit pull $QUARRY_UPSTREAM 2>&1 | tail -1 | tee /dev/stderr | grep -qvE 'refused|failed|error|cannot'"; do
n=$((n+1)); [ "$n" -lt 8 ] || { echo "deploy: pull from $QUARRY_UPSTREAM kept failing" >&2; exit 1; }
sleep 8
done
fi
work=$(mktemp -d /tmp/quarry-deploy.XXXXXX)
trap 'rm -rf "$work"' EXIT
chown quarry:quarry "$work"
sudo -u quarry sh -c "cd $work && $BIN/zit open trunk --store=$STORE/.zit >/dev/null"
tip=$(cd "$work" && sudo -u quarry $BIN/zit status | awk '/^checkout:/ {print substr($2,1,12)}')
# This script is BOTH tracked here and installed at /srv/quarry/deploy.sh,
# and the installed copy is the one that runs. So the two can drift, and
# on 2026-09-19 they did: the installed copy was edited by hand to build
# and install the CI worker, with no review possible because the file was
# not in the repository. That is the same silence that let the worker go
# fifteen days and one protocol migration without being rebuilt.
#
# Warn, never fail. A deploy that refuses to run because its own script
# disagrees with the tree is worse than one that tells you they disagree
# — and this script must keep working when the tree's copy is the broken
# one, which is exactly when you need to deploy.
if [ -f "$work/scripts/deploy.sh" ] && ! cmp -s "$0" "$work/scripts/deploy.sh"; then
echo "deploy: WARNING — $0 differs from scripts/deploy.sh in trunk $tip" >&2
echo "deploy: diff -u $0 $work/scripts/deploy.sh" >&2
echo "deploy: install -m 0755 $work/scripts/deploy.sh $0 # once the tree's copy is the right one" >&2
fi
# The build image is the Go that go.mod names — one pin, not two. And
# every deploy looks for a newer stable Go: quarry is meant to track the
# latest (e3dfc93f), and a warning here is the one place an operator
# reads on every release. Warn, never fail, like the check above.
gover=$(sed -n 's/^go //p' "$work/go.mod")
latest=$(curl -s -m 10 'https://go.dev/dl/?mode=json' | grep -o '"version": *"go[0-9.]*"' | head -1 | grep -o '[0-9][0-9.]*')
if [ -n "$latest" ] && [ "$latest" != "$gover" ]; then
echo "deploy: WARNING — go.mod builds with Go $gover; Go $latest is the latest stable. Bump it: go mod edit -go=$latest && go mod tidy" >&2
fi
echo "deploy: building trunk $tip with Go $gover"
docker run --rm -v "$work":/src -v "$ZITREL":/zitrel:ro -w /src -e GOTOOLCHAIN=auto -e ZIT_RELEASE_DIR=/zitrel \
-e VERSION="0.1.0-beta+$tip" -e PREFIX=/src/bin "golang:$gover" sh -c "scripts/build-linux.sh" 2>&1 | grep -v "^go: downloading" | tail -2
[ -x "$work/bin/quarry" ] || { echo "deploy: build produced no binary" >&2; exit 1; }
cp -p "$BIN/quarry" "$BIN/quarry.prev" 2>/dev/null || true
install -m 0755 "$work/bin/quarry" "$BIN/quarry.new" && mv -f "$BIN/quarry.new" "$BIN/quarry"
install -m 0755 "$work/bin/zit" "$BIN/zit"
systemctl restart quarry.service
ok=
for i in $(seq 1 30); do
if curl -sf -m 3 -o /dev/null http://127.0.0.1:8950/; then ok=1; break; fi
sleep 2
done
if [ "$ok" != 1 ]; then
echo "deploy: new binary did not answer in 60s - rolling back" >&2
mv -f "$BIN/quarry.prev" "$BIN/quarry" && systemctl restart quarry.service
exit 1
fi
# The port answering is not the forge answering. After a restart it
# registers its repositories one by one — 65-75s for two dozen — and
# refuses every sync and every repository page as "still starting" for
# that whole window, while / answers 200. This script used to print
# success at the top of that window, so the operator's last signal was
# "serving" while the forge was down for another minute (b81ae623).
# So: wait for ready, say how long it took, and refuse to call a forge
# that never gets there healthy.
started=$(date +%s)
ready=
for i in $(seq 1 150); do
if curl -sf -m 3 http://127.0.0.1:8950/api/v1/version | grep -q '"ready":true'; then ready=1; break; fi
sleep 2
done
waited=$(( $(date +%s) - started ))
if [ "$ready" != 1 ]; then
echo "deploy: forge is up but not ready after ${waited}s; still starting: $(curl -s -m 3 http://127.0.0.1:8950/api/v1/version | grep -o '"starting":\[[^]]*\]')" >&2
echo "deploy: NOT rolling back — the binary answers; registration is what is slow. Watch: journalctl -u quarry -f" >&2
exit 1
fi
echo "deploy: zeitforge.app serving $(curl -s http://127.0.0.1:8950/ | grep -o 'quarry 0[^<]*' | head -1 | sed 's/+/+/') — ready after ${waited}s"
# quarry's own CI runs on `quarry runner` (quarry-runner.service, unit in
# /etc/systemd/system, token in /etc/quarry-runner.env). It is this same
# binary, so it restarts onto the new build — after the forge is ready,
# because it polls through the forge's CI bridge. The NATS worker it
# replaced (ci/worker) is gone; a host that never installed the runner
# unit skips this quietly.
if systemctl cat quarry-runner.service >/dev/null 2>&1; then
systemctl restart quarry-runner.service && echo "deploy: quarry runner restarted"
fi
# 6. warm the edge: every deploy renames the hashed assets, so the first
# visitor at each Cloudflare location would otherwise pay the miss.
warm() { curl -s -m 10 -o /dev/null -H "Accept-Encoding: gzip" "https://zeitforge.app$1"; }
warm / ; warm /start
for a in $(curl -s -m 10 https://zeitforge.app/ | grep -o "/assets/[^\"]*" | sort -u); do warm "$a"; done
echo "deploy: edge warmed"