#!/bin/sh
# Nightly backup of everything on this host that is not derived
# (ticket d49b1bf22c). Derived indexes rebuild from the stores and are
# not copied.
#
# stores every .zit store under /srv/quarry/repos (tar, consistent
# because a zit store is one append-only file per repository).
# A repository that follows a Git upstream also has zit's
# import files beside its store (.zit-import-map, and an
# interrupted import's .zit-import-pending/-completed): without
# them the store cannot take its next pull. They are copied
# with the store under the lock the forge's pull holds
# (.quarry-git.lock), so never in the middle of an import.
# oltp /srv/quarry/data/oltp.db via SQLite's online backup API,
# so a copy taken while the forge writes is still a whole database
# notify /srv/notify/notify.db, the same way
# dagnats /srv/dagnats/data (run history), without the TELEMETRY
# stream: a gigabyte of traces SigNoz already holds, written
# to all night, so tar never read it whole
#
# Kept for KEEP_DAYS under /srv/backups/<stamp>/, newest listed in
# /srv/backups/latest, and for OFFSITE_KEEP_DAYS in the R2 bucket. The
# disk keeps few because each set is a full copy of every store; the
# bucket is the history. A run that fails leaves no set behind.
set -eu
ROOT=/srv/backups
KEEP_DAYS="${KEEP_DAYS:-3}"
OFFSITE_KEEP_DAYS="${OFFSITE_KEEP_DAYS:-14}"
stamp=$(date -u +%Y%m%dT%H%M%SZ)
dst="$ROOT/$stamp"
mkdir -p "$dst"
# SQLite online backups: a consistent snapshot without stopping anything.
sqlite3 /srv/quarry/data/oltp.db ".backup '$dst/oltp.db'"
[ -f /srv/notify/notify.db ] && sqlite3 /srv/notify/notify.db ".backup '$dst/notify.db'"
# Stores: the .zit file is the whole repository, except for a Git
# mirror, whose import files are copied with it into a staging tree
# while its pull lock is held.
stage=$(mktemp -d "$ROOT/.stage.XXXXXX")
trap 'rm -rf "$stage"; [ -e "$dst/SHA256SUMS" ] || rm -rf "$dst"' EXIT
for map in /srv/quarry/repos/*/.zit-import-map; do
[ -e "$map" ] || continue
repo=$(dirname "$map"); name=$(basename "$repo")
mkdir -p "$stage/$name"
flock "$repo/.quarry-git.lock" sh -c 'cd "$1" && cp -a .zit .zit-import-* "$2"/' sh "$repo" "$stage/$name"
done
plain=$(cd /srv/quarry/repos && for z in */.zit; do [ -e "$(dirname "$z")/.zit-import-map" ] || echo "$z"; done)
mirrors=$(cd "$stage" && ls -A)
tar -czf "$dst/stores.tar.gz" -C /srv/quarry/repos $plain ${mirrors:+-C "$stage" $mirrors}
# JetStream appends while tar reads: tar's exit 1 ("file changed as we
# read it") is a torn tail block the server discards on restore; 2 is a
# real failure.
tar -C /srv/dagnats -czf "$dst/dagnats-data.tar.gz" --warning=no-file-changed \
--exclude='data/jetstream/*/streams/TELEMETRY' data || [ $? -eq 1 ]
( cd "$dst" && sha256sum * > SHA256SUMS )
ln -sfn "$dst" "$ROOT/latest"
# Verify: the copied databases open and count what the live ones count.
live=$(sqlite3 /srv/quarry/data/oltp.db "select count(*) from account")
copy=$(sqlite3 "$dst/oltp.db" "select count(*) from account")
[ "$live" = "$copy" ] || { echo "backup: oltp copy disagrees ($copy accounts, live $live)" >&2; exit 1; }
sqlite3 "$dst/oltp.db" "pragma integrity_check" | grep -qx ok || { echo "backup: oltp copy failed integrity check" >&2; exit 1; }
# Offsite: the whole set to the R2 bucket, when its keys are present.
# rclone speaks S3 to R2; the config is built from the environment so
# no key is written to disk.
if [ -n "${BACKUP_S3_BUCKET:-}" ] && command -v rclone >/dev/null; then
export RCLONE_CONFIG_R2_TYPE=s3 RCLONE_CONFIG_R2_PROVIDER=Cloudflare \
RCLONE_CONFIG_R2_ENDPOINT="$BACKUP_S3_ENDPOINT" RCLONE_CONFIG_R2_REGION=auto \
RCLONE_CONFIG_R2_ACCESS_KEY_ID="$BACKUP_S3_ACCESS_KEY" RCLONE_CONFIG_R2_SECRET_ACCESS_KEY="$BACKUP_S3_SECRET_KEY"
rclone copy --quiet "$dst" "r2:$BACKUP_S3_BUCKET/nightly/$stamp/" ||
{ echo "backup: offsite copy to $BACKUP_S3_BUCKET/nightly/$stamp failed" >&2; exit 1; }
echo "backup: copied offsite to $BACKUP_S3_BUCKET/nightly/$stamp"
rclone delete --quiet --min-age "${OFFSITE_KEEP_DAYS}d" "r2:$BACKUP_S3_BUCKET/nightly/" 2>/dev/null || true
else
echo "backup: no BACKUP_S3_BUCKET in the environment — offsite copy skipped"
fi
# Rotate.
find "$ROOT" -maxdepth 1 -type d -name '2*' -mtime +"$KEEP_DAYS" -exec rm -rf {} +
echo "backup: $dst ($(du -sh "$dst" | cut -f1)), oltp $copy accounts, integrity ok"