#!/bin/sh
# release.sh [--dry-run] [--out DIR] X.Y.Z
#
# Cut quarry's release X.Y.Z from the checkout this runs in (RFC 0020
# §releases): build, package, sum, describe, publish. ci/release.yml runs
# it on the tag version-X.Y.Z; deploy.sh --release X.Y.Z installs what it
# publishes, so the layout below is the one deploy.sh reads (docs/running.md,
# "Releases and rollback").
#
# 1. refuse unless CHANGELOG.md's newest release heading is "## X.Y.Z",
# the checkout carries version-X.Y.Z itself (the rule the tag
# trigger and the version stamp use, scripts/version-stamp.sh), and no
# tracked file is changed and no untracked one is present
# 2. build with scripts/build-linux.sh: linux-amd64 natively with the five
# runner downloads; linux-arm64 only with RELEASE_ARM64=1, by a cross C
# toolchain, and checked to be ELF for aarch64
# 3. package into OUT (default $TMPDIR/quarry-release-X.Y.Z), which must
# be absent or empty; nothing in it is ever removed:
# quarry-X.Y.Z-linux-<amd64|arm64>.tar.gz one directory,
# quarry-X.Y.Z-linux-<arch>/, holding quarry, zit, CHANGELOG.md, LICENSE
# quarry-runner-<os>-<arch>[.exe] the runner downloads
# quarry-X.Y.Z-src.tar.gz the tagged tree (zit tarball)
# SHA256SUMS one line per file above
# release-manifest.json zit's schema_version 1
# 4. verify SHA256SUMS against the files on disk
# 5. publish each file as releases/X.Y.Z/<file> with `zit uv push`, as
# QUARRY_USER with QUARRY_TOKEN: SHA256SUMS first, the manifest last,
# so a release whose manifest is there is whole
#
# --dry-run does all of it but step 5.
#
# Re-running is safe. When the forge already holds releases/X.Y.Z's
# manifest, it is fetched: if it names this check-in the release is
# published and nothing is built or pushed (exit 0); if it names another,
# the release is refused (exit 1). Otherwise every file is pushed again:
# `zit uv push` skips a name the forge holds with the same hash, and a
# rebuilt file with other bytes replaces the half-published one, since its
# time is later.
#
# Environment:
# QUARRY_USER, QUARRY_TOKEN the run token the release job receives (RFC 0021
# §runs). The token is taken out of the environment before
# anything runs and handed only to `zit uv` as ZIT_PASSWORD,
# never as an argument, which every user on the machine can read
# QUARRY_TAG the tag that started the run; when set, it must be version-X.Y.Z
# RELEASE_URL the repository to publish to (default https://zeitforge.app/r/quarry)
# RELEASE_ARM64 1 to build linux-arm64 too; otherwise the manifest lists it
# as omitted
# ARM64_CC, ARM64_CXX the cross C toolchain for linux-arm64
# (default aarch64-linux-gnu-gcc, aarch64-linux-gnu-g++)
# ZIT_RELEASE_DIR, GITHUB_TOKEN how build-linux.sh gets the pinned zit
# RELEASE_BUILD a command run as `$RELEASE_BUILD GOARCH PREFIX` (VERSION
# and NO_RUNNERS exported) in place of build-linux.sh, for
# the tests
set -eu
# The token, out of the environment before anything else runs: no build,
# no fetch and no zit but the uv calls below can see it.
token=${QUARRY_TOKEN:-}
unset QUARRY_TOKEN ZIT_PASSWORD
usage="usage: release.sh [--dry-run] [--out DIR] X.Y.Z"
dry=; out=; ver=
while [ $# -gt 0 ]; do
case "$1" in
--dry-run) dry=1 ;;
--out) out=${2:?$usage}; shift ;;
-*) echo "$usage" >&2; exit 2 ;;
*) [ -z "$ver" ] || { echo "$usage" >&2; exit 2; }; ver=$1 ;;
esac
shift
done
printf '%s\n' "$ver" | grep -Eqx '[0-9]+\.[0-9]+\.[0-9]+' || { echo "$usage" >&2; exit 2; }
tag=version-$ver
url=${RELEASE_URL:-https://zeitforge.app/r/quarry}
arm64=${RELEASE_ARM64:-}
die() { echo "release: $*" >&2; exit 1; }
sha256() {
if command -v sha256sum >/dev/null 2>&1; then sha256sum "$@"; else shasum -a 256 "$@"; fi
}
size() { wc -c <"$1" | tr -d ' '; }
# uv runs zit's unversioned verbs as the run's login, the token in zit's
# environment alone.
uv() { ZIT_PASSWORD=$token zit uv "$@" --user "$QUARRY_USER"; }
# ---- 1. is this check-in release X.Y.Z, and clean ----------------------
[ -f CHANGELOG.md ] && [ -f scripts/version-stamp.sh ] || die "run this at the root of a quarry checkout"
if [ -n "${QUARRY_TAG:-}" ] && [ "$QUARRY_TAG" != "$tag" ]; then
die "the run was started by the tag $QUARRY_TAG, not $tag"
fi
# The release is the changelog's newest heading: the stamp prints X.Y.Z
# for a check-in tagged version-X.Y.Z only when that heading says X.Y.Z,
# so a release is never cut under an older or newer section's notes.
newest=$(sh scripts/version-stamp.sh CHANGELOG.md local "$tag") || die "CHANGELOG.md names no release: a release says what changed"
[ "$newest" = "$ver" ] || die "CHANGELOG.md's newest heading is not \"## $ver\": add that section above the others, land it, and tag that check-in"
checkin=$(zit info current 2>/dev/null | head -1) || true
case "$checkin" in
*[!0-9a-f]* | "") die "not a zit checkout: no check-in to release (the release runner checks out with --checkout zit)" ;;
esac
[ "${#checkin}" -eq 40 ] || [ "${#checkin}" -eq 64 ] || die "zit info current named $checkin, not a full hash"
# Which tags the check-in carries itself, read as build-linux.sh and
# deploy.sh read them; any read that fails refuses, never guesses.
raw=$(zit tag list --raw "$checkin") || die "could not read $checkin's tags"
info=$(zit info "$checkin") || die "could not read $checkin's parent"
parent=$(printf '%s\n' "$info" | sed -n 's/^parent: *//p')
praw=
if [ -n "$parent" ]; then
praw=$(zit tag list --raw "$parent") || die "could not read the parent's tags"
fi
plain=$(zit tag list "$checkin") || die "could not read $checkin's tags"
stamp=$(printf '%s\n--\n%s\n--\n%s\n' "$raw" "$praw" "$plain" | sh scripts/version-stamp.sh CHANGELOG.md "$checkin" -)
[ "$stamp" = "$ver" ] || die "check-in $(printf '%.12s' "$checkin") does not carry $tag itself (it would be stamped $stamp)"
dirty=$(zit changes) || die "zit changes failed"
[ -z "$dirty" ] || die "the checkout has changes; a release is the tagged tree as it is:
$dirty"
extra=$(zit extras) || die "zit extras failed"
[ -z "$extra" ] || die "the checkout has untracked files; a release is the tagged tree as it is:
$extra"
out=${out:-${TMPDIR:-/tmp}/quarry-release-$ver}
if [ -e "$out" ]; then
[ -d "$out" ] && [ -z "$(ls -A "$out")" ] || die "$out is not empty; name an empty or new directory with --out"
fi
if [ -n "$arm64" ]; then
# Asked before anything builds: a missing toolchain fails in seconds.
for c in "${ARM64_CC:=aarch64-linux-gnu-gcc}" "${ARM64_CXX:=aarch64-linux-gnu-g++}"; do
command -v "$c" >/dev/null 2>&1 || die "RELEASE_ARM64 is set and $c is not on PATH; install the linux-arm64 cross toolchain or leave RELEASE_ARM64 unset"
done
fi
work=$(mktemp -d "${TMPDIR:-/tmp}/quarry-release-build.XXXXXX")
# rel is the release as it is assembled: a sibling of OUT, on its
# filesystem, renamed into place whole at the end.
mkdir -p "$(dirname "$out")"
rel=$(mktemp -d "$(dirname "$out")/.quarry-release-$ver.XXXXXX")
trap 'rm -rf "$work" ${rel:+"$rel"}' EXIT
trap 'exit 130' INT TERM HUP
# ---- already published? -------------------------------------------------
# Asked before building: a published release is never rebuilt over, and
# one published from another check-in is not this release.
if [ -z "$dry" ]; then
[ -n "${QUARRY_USER:-}" ] && [ -n "$token" ] ||
die "publishing needs QUARRY_USER and QUARRY_TOKEN, the run token a release job receives; --dry-run builds without them"
held=$(uv list "$url") ||
die "could not list $url's unversioned files as $QUARRY_USER (zit $(zit version | awk '{print $2}') on PATH; uv needs 0.3.36)"
manifest=releases/$ver/release-manifest.json
if printf '%s\n' "$held" | grep -qxF "$manifest"; then
uv pull "$url" "$manifest" -o "$work/held" >/dev/null || die "could not fetch $url's $manifest"
# The release's own check-in is the first fossil_check_in; source repeats it.
was=$(sed -n 's/.*"fossil_check_in": *"\([0-9a-f]*\)".*/\1/p' "$work/held/$manifest" | head -1)
[ "$was" = "$checkin" ] ||
die "$url already holds release $ver from check-in $(printf '%.12s' "${was:-none}"), not $(printf '%.12s' "$checkin"); a published release is not replaced"
echo "release: $url already holds release $ver from this check-in; nothing published"
exit 0
fi
fi
# ---- 2. build ------------------------------------------------------------
# The job's HOME is its checkout: Go's caches go to the build directory,
# so neither dirties the tree nor outlives the run (-modcacherw lets the
# trap remove the module cache).
export GOCACHE="${GOCACHE:-$work/gocache}" GOPATH="${GOPATH:-$work/gopath}"
export GOFLAGS="${GOFLAGS:+$GOFLAGS }-modcacherw"
export VERSION="$ver"
build() { # build GOARCH PREFIX; the runner downloads come with amd64 alone
NO_RUNNERS=; [ "$1" = amd64 ] || NO_RUNNERS=1
export NO_RUNNERS
if [ -n "${RELEASE_BUILD:-}" ]; then
$RELEASE_BUILD "$1" "$2"
elif [ "$1" = arm64 ]; then
GOARCH=arm64 PREFIX=$2 CC=$ARM64_CC CXX=$ARM64_CXX sh scripts/build-linux.sh
else
GOARCH=$1 PREFIX=$2 sh scripts/build-linux.sh
fi
}
arches=amd64
[ -z "$arm64" ] || arches="amd64 arm64"
for arch in $arches; do
echo "release: building quarry $ver for linux-$arch"
build "$arch" "$work/$arch" || die "the linux-$arch build failed"
for b in quarry zit; do
[ -s "$work/$arch/$b" ] || die "the linux-$arch build made no $b"
done
done
# A cross build is not run before publishing, so it is at least the right
# kind of file: ELF (7f 45 4c 46) for aarch64 (e_machine 183, b7 00).
if [ -n "$arm64" ]; then
for b in quarry zit; do
head=$(od -An -tx1 -N20 "$work/arm64/$b" | tr -d ' \n')
case "$head" in
7f454c46*b700) ;;
*) die "the linux-arm64 $b is not an ELF binary for aarch64" ;;
esac
done
fi
# The runner downloads are pure Go, built once; the set is checked as
# deploy.sh checks one, against this tree's list.
sh scripts/runner-check.sh "$work/amd64/runner" scripts/build-linux.sh >"$work/runner-check" ||
die "the runner downloads are not a complete set: $(cat "$work/runner-check")"
# The forge build that can run here must say it is this release, as
# deploy.sh --release will insist.
case "$(uname -s)-$(uname -m)" in
Linux-x86_64 | Linux-amd64)
got=$("$work/amd64/quarry" version) || die "the linux-amd64 quarry does not run"
[ "$got" = "quarry $ver" ] || die "the linux-amd64 quarry says \"$got\", not quarry $ver" ;;
esac
# ---- 3. package ----------------------------------------------------------
for arch in $arches; do
top=quarry-$ver-linux-$arch
mkdir -p "$work/pkg/$top"
install -m 0755 "$work/$arch/quarry" "$work/$arch/zit" "$work/pkg/$top/"
install -m 0644 CHANGELOG.md LICENSE "$work/pkg/$top/"
# COPYFILE_DISABLE: macOS tar adds ._ files, which deploy.sh refuses.
(cd "$work/pkg" && COPYFILE_DISABLE=1 tar -cf - "$top") | gzip -n >"$rel/$top.tar.gz"
done
for f in "$work/amd64/runner"/quarry-runner-*; do
install -m 0755 "$f" "$rel/"
done
zit tarball "$checkin" "$rel/quarry-$ver-src.tar.gz" --name "quarry-$ver-src" >/dev/null ||
die "zit tarball of $checkin failed"
(cd "$rel" && sha256 quarry-*) >"$work/SHA256SUMS"
mv "$work/SHA256SUMS" "$rel/SHA256SUMS"
# ---- the manifest -------------------------------------------------------
sum_of() { awk -v f="$1" '$2 == f || $2 == "*" f {print $1}' "$rel/SHA256SUMS"; }
artifact() { # artifact NAME TARGET QUALIFICATION
printf ' {"name": "%s", "target": "%s", "bytes": %s, "sha256": "%s", "qualification": "%s"}' \
"$1" "$2" "$(size "$rel/$1")" "$(sum_of "$1")" "$3"
}
native="built natively on linux-amd64 by scripts/build-linux.sh; quarry version prints quarry $ver"
crossq="cross-built on linux-amd64 by scripts/build-linux.sh with a linux-arm64 C toolchain; ELF for aarch64, not run before publishing"
case "$(uname -s)-$(uname -m)" in
Linux-x86_64 | Linux-amd64) ;;
*) native="built by scripts/build-linux.sh; not run before publishing" ;;
esac
runnerq="cross-compiled pure Go (CGO_ENABLED=0) by scripts/build-linux.sh; the set is complete (scripts/runner-check.sh)"
nobuild="no forge build: the DuckDB bindings do not cross-link from Linux, and the forge is built on Linux (scripts/build-linux.sh); its runner download is published"
src=quarry-$ver-src.tar.gz
{
echo '{'
echo ' "schema_version": 1,'
echo ' "release": {'
printf ' "version": "%s",\n "date": "%s",\n "fossil_check_in": "%s",\n "fossil_tag": "%s"\n' \
"$ver" "$(date -u +%Y-%m-%d)" "$checkin" "$tag"
echo ' },'
printf ' "source": {"name": "%s", "target": "source", "bytes": %s, "sha256": "%s", "fossil_check_in": "%s", "fossil_tag": "%s", "qualification": "%s"},\n' \
"$src" "$(size "$rel/$src")" "$(sum_of "$src")" "$checkin" "$tag" "zit tarball of the tagged check-in, from which every artifact here was built"
echo ' "artifacts": ['
artifact "quarry-$ver-linux-amd64.tar.gz" linux-amd64 "$native"
if [ -n "$arm64" ]; then
echo ','
artifact "quarry-$ver-linux-arm64.tar.gz" linux-arm64 "$crossq"
fi
for f in "$rel"/quarry-runner-*; do
n=${f##*/}
t=${n#quarry-runner-}
echo ','
artifact "$n" "runner-${t%.exe}" "$runnerq"
done
echo
echo ' ],'
echo ' "omitted_targets": ['
if [ -z "$arm64" ]; then
printf ' {"target": "linux-arm64", "reason": "not built for this release: the linux-arm64 forge build is opt-in (RELEASE_ARM64=1) until its cross toolchain is qualified"},\n'
fi
for t in darwin-arm64 darwin-amd64 windows-amd64; do
[ "$t" = darwin-arm64 ] || echo ','
printf ' {"target": "%s", "reason": "%s"}' "$t" "$nobuild"
done
echo
echo ' ],'
echo ' "qualification": {'
printf ' "linux-amd64": ["%s"],\n' "$native"
[ -z "$arm64" ] || printf ' "linux-arm64": ["%s"],\n' "$crossq"
printf ' "runner": ["%s"]\n' "$runnerq"
echo ' }'
echo '}'
} >"$rel/release-manifest.json"
# ---- 4. verify -----------------------------------------------------------
(cd "$rel" && sha256 -c SHA256SUMS) >/dev/null 2>&1 || die "the release does not match its SHA256SUMS"
for f in "$rel"/*; do
n=${f##*/}
case "$n" in SHA256SUMS | release-manifest.json) continue ;; esac
[ -n "$(sum_of "$n")" ] || die "SHA256SUMS has no line for $n"
done
# Into OUT only now, by one rename, and only if OUT is still absent or
# an empty directory (removed first, so the rename neither nests nor
# overwrites).
if [ -e "$out" ]; then
if ! { [ -d "$out" ] && [ -z "$(ls -A "$out")" ] && rmdir "$out" 2>/dev/null; }; then
die "$out is not empty; name an empty or new directory with --out"
fi
fi
[ ! -e "$out" ] || die "$out appeared while the release was built; nothing moved into it"
mv "$rel" "$out" || die "could not rename the release into $out"
chmod 0755 "$out"
rel=
if [ -n "$dry" ]; then
echo "release: dry run, nothing published; release $ver is in $out"
exit 0
fi
# ---- 5. publish ----------------------------------------------------------
push() {
echo "release: publishing releases/$ver/$1"
uv push "$url" "$out/$1" --as "releases/$ver/$1" ||
die "zit uv push of $1 failed; re-run to finish: what is already there is skipped"
}
push SHA256SUMS
for f in "$out"/quarry-*; do
push "${f##*/}"
done
push release-manifest.json
echo "release: published $ver to $url/releases ($(printf '%.12s' "$checkin"))"