// releases.go is RFC 0020 rung 5: a repository's releases, read from its
// unversioned files. A release is a folder, releases/<name>/, whatever
// the repository's tags look like; its manifest, when it publishes one,
// is zit's release-manifest.json, and its notes are the CHANGELOG.md
// section whose heading names it.
package api
import (
"bytes"
"context"
"encoding/hex"
"encoding/json"
"fmt"
"log"
"regexp"
"sort"
"strconv"
"strings"
"sync"
"time"
"craftdesign.group/quarry/internal/ops"
"craftdesign.group/quarry/internal/s3"
"craftdesign.group/quarry/internal/uv"
)
const (
releasesPrefix = "releases/"
// releaseManifestName and releaseSumsName are the two files a
// release folder may describe itself with.
releaseManifestName = "release-manifest.json"
releaseSumsName = "SHA256SUMS"
// releaseManifestMax bounds a manifest read through the forge. zit's
// is a few kilobytes; anything near this is not one. It is checked
// against the catalog's size; the read itself is capped by s3's
// maxForgeObject. The two can disagree only for an object changed
// in the bucket behind the forge's back, and then the read's cap is
// the one that holds.
releaseManifestMax = 1 << 20
)
type releaseListIn struct {
Repo string `json:"repo" validate:"required,max=64"`
// Name, when given, answers that one release alone: a folder name,
// never a path.
Name string `json:"name" validate:"max=400"`
}
type releaseSummaryIn struct {
Repo string `json:"repo" validate:"required,max=64"`
}
// releaseSummaryOut is the Files page's block: how many releases and
// which is newest, from the catalog alone.
type releaseSummaryOut struct {
Count int `json:"count"`
Latest string `json:"latest,omitempty"`
Mtime int64 `json:"mtime,omitempty"`
}
// checkReleaseName refuses what cannot be a folder under releases/.
func checkReleaseName(name string) error {
if name == "" || name == "." || name == ".." || strings.ContainsAny(name, "/\\") {
return fmt.Errorf("%q is not a release name: %w", name, ops.ErrInvalid)
}
return nil
}
type releaseListOut struct {
// Bound and Note are uv.list's: no bucket, no releases.
Bound bool `json:"bound"`
Note string `json:"note,omitempty"`
Releases []release `json:"releases"`
}
// release is one folder under releases/.
type release struct {
Name string `json:"name"`
// Newest marks the first group, the one the order puts on top.
Newest bool `json:"newest"`
// Mtime is the newest file's, which orders groups whose names are
// not versions.
Mtime int64 `json:"mtime"`
Files []uv.File `json:"files"`
// Sums names the group's SHA256SUMS, when it has one.
Sums string `json:"sums,omitempty"`
Manifest *releaseManifest `json:"manifest,omitempty"`
ManifestErr string `json:"manifestErr,omitempty"`
// Notes is the CHANGELOG.md section whose heading names the release.
Notes *releaseNotes `json:"notes,omitempty"`
}
type releaseNotes struct {
Heading string `json:"heading"`
HTML string `json:"html"`
}
// releaseManifest is what the page shows of zit's schema version 1.
type releaseManifest struct {
Version string `json:"version"`
Date string `json:"date,omitempty"`
Checkin string `json:"checkin,omitempty"`
Tag string `json:"tag,omitempty"`
Artifacts []releaseArtifact `json:"artifacts"`
Source *releaseArtifact `json:"source,omitempty"`
Omitted []releaseOmitted `json:"omitted,omitempty"`
Qualification []releaseQualification `json:"qualification,omitempty"`
}
type releaseArtifact struct {
Name string `json:"name"`
Target string `json:"target"`
Size int64 `json:"size"`
SHA256 string `json:"sha256"`
Qualification string `json:"qualification,omitempty"`
// File is the unversioned name the artifact is published under, when
// the group holds it.
File string `json:"file,omitempty"`
// SumsSHA256 is what the group's SHA256SUMS says of this file, set
// only when that differs from the manifest's sha256: two files that
// vouch for one artifact disagree, and the page says so rather than
// show the manifest's word alone.
SumsSHA256 string `json:"sumsSha256,omitempty"`
}
// releaseOmitted is a target the release was not built for, and why
// when the manifest says.
type releaseOmitted struct {
Target string `json:"target"`
Reason string `json:"reason,omitempty"`
}
type releaseQualification struct {
Target string `json:"target"`
Notes []string `json:"notes"`
}
// manifestArtifact is one entry as zit writes it. zit says bytes where
// the RFC said size and fossil_check_in where it said checkin; both
// spellings are read.
type manifestArtifact struct {
Name string `json:"name"`
Target string `json:"target"`
Bytes *int64 `json:"bytes"`
Size *int64 `json:"size"`
SHA256 string `json:"sha256"`
Qualification json.RawMessage `json:"qualification"`
FossilCheckIn string `json:"fossil_check_in"`
Checkin string `json:"checkin"`
}
type manifestFile struct {
SchemaVersion int `json:"schema_version"`
Artifacts []manifestArtifact `json:"artifacts"`
OmittedTargets []json.RawMessage `json:"omitted_targets"`
Qualification map[string]json.RawMessage `json:"qualification"`
Release struct {
Version string `json:"version"`
Date string `json:"date"`
FossilCheckIn string `json:"fossil_check_in"`
Checkin string `json:"checkin"`
FossilTag string `json:"fossil_tag"`
Tag string `json:"tag"`
} `json:"release"`
Source *manifestArtifact `json:"source"`
}
func (a manifestArtifact) view() releaseArtifact {
out := releaseArtifact{Name: a.Name, Target: a.Target, SHA256: a.SHA256,
Qualification: strings.Join(jsonWords(a.Qualification), "; ")}
switch {
case a.Bytes != nil:
out.Size = *a.Bytes
case a.Size != nil:
out.Size = *a.Size
}
return out
}
// jsonWords reads a manifest value meant for people: a string, a list
// of strings, or anything else as its JSON text.
func jsonWords(raw json.RawMessage) []string {
if len(raw) == 0 || string(raw) == "null" {
return nil
}
var s string
if json.Unmarshal(raw, &s) == nil {
return []string{s}
}
var list []string
if json.Unmarshal(raw, &list) == nil {
return list
}
var buf bytes.Buffer
if json.Compact(&buf, raw) != nil {
return nil
}
return []string{buf.String()}
}
// parseReleaseManifest reads zit's schema, tolerating missing keys. A
// check-in that is not a hash is dropped, so the page never builds a
// link from whatever else a manifest says there.
func parseReleaseManifest(raw []byte, folder string) (*releaseManifest, error) {
var m manifestFile
if err := json.Unmarshal(raw, &m); err != nil {
return nil, fmt.Errorf("release-manifest.json is not JSON: %v", err)
}
if m.SchemaVersion != 1 {
return nil, fmt.Errorf("release-manifest.json is schema_version %d; this forge reads 1", m.SchemaVersion)
}
out := &releaseManifest{Version: m.Release.Version, Date: m.Release.Date, Tag: m.Release.FossilTag}
if out.Version == "" {
out.Version = folder
}
if out.Tag == "" {
out.Tag = m.Release.Tag
}
for _, a := range m.Artifacts {
out.Artifacts = append(out.Artifacts, a.view())
}
checkins := []string{m.Release.FossilCheckIn, m.Release.Checkin}
if m.Source != nil {
s := m.Source.view()
out.Source = &s
checkins = append(checkins, m.Source.FossilCheckIn, m.Source.Checkin)
}
for _, c := range checkins {
if out.Checkin == "" && hexTail.MatchString(c) {
out.Checkin = c
}
}
for _, raw := range m.OmittedTargets {
var o releaseOmitted
if json.Unmarshal(raw, &o) == nil && o.Target != "" {
out.Omitted = append(out.Omitted, releaseOmitted{Target: o.Target, Reason: o.Reason})
continue
}
for _, w := range jsonWords(raw) {
out.Omitted = append(out.Omitted, releaseOmitted{Target: w})
}
}
targets := make([]string, 0, len(m.Qualification))
for t := range m.Qualification {
targets = append(targets, t)
}
sort.Strings(targets)
for _, t := range targets {
out.Qualification = append(out.Qualification, releaseQualification{Target: t, Notes: jsonWords(m.Qualification[t])})
}
return out, nil
}
// withFiles is m with each artifact linked to the file the folder holds
// under its name, when it holds one, and checked against the folder's
// SHA256SUMS. group maps the folder's file names to their full names;
// sums maps a file name to the digest SHA256SUMS gives it. m is shared
// through the cache and not changed.
func (m *releaseManifest) withFiles(group, sums map[string]string) *releaseManifest {
link := func(a releaseArtifact) releaseArtifact {
a.File = group[a.Name]
if want, ok := sums[a.Name]; ok && !strings.EqualFold(want, a.SHA256) {
a.SumsSHA256 = want
}
return a
}
out := *m
out.Artifacts = make([]releaseArtifact, len(m.Artifacts))
for i, a := range m.Artifacts {
out.Artifacts[i] = link(a)
}
if m.Source != nil {
s := link(*m.Source)
out.Source = &s
}
return &out
}
// parseSums reads a SHA256SUMS file as sha256sum writes it: a 64-digit
// hex digest, a space, then a space or a '*' (binary mode), then the
// name. Lines in any other shape are skipped, and a name given twice
// keeps its first digest. Names are taken as the folder's base names,
// which is how a release's SHA256SUMS lists its own files.
func parseSums(raw []byte) map[string]string {
out := map[string]string{}
for _, line := range strings.Split(string(raw), "\n") {
line = strings.TrimRight(line, "\r")
if len(line) < 67 || (line[65] != ' ' && line[65] != '*') || line[64] != ' ' {
continue
}
digest, name := line[:64], line[66:]
if _, err := hex.DecodeString(digest); err != nil || name == "" {
continue
}
name = strings.TrimPrefix(name, "./")
if _, seen := out[name]; !seen {
out[name] = strings.ToLower(digest)
}
}
return out
}
// manifestCache holds parsed manifests by the file's content hash, so a
// page view reads the bucket only for a manifest it has not seen. The
// key is the content, so an entry cannot go stale; the bound is on
// entries, oldest out first.
var manifestCache = struct {
sync.Mutex
entries map[string]manifestEntry
order []string
}{entries: map[string]manifestEntry{}}
const manifestCacheEntries = 128
type manifestEntry struct {
m *releaseManifest
sums map[string]string
err string
}
// releaseManifestOf is the parsed manifest f holds, from the cache or
// the bucket; errText says why there is none.
func releaseManifestOf(ctx context.Context, bucket s3.Bucket, repo, folder string, f uv.File) (*releaseManifest, string) {
e := releaseFileOf(ctx, bucket, repo, f, "m\x00"+folder, func(raw []byte) manifestEntry {
m, err := parseReleaseManifest(raw, folder)
if err != nil {
return manifestEntry{err: err.Error()}
}
return manifestEntry{m: m}
})
return e.m, e.err
}
// releaseSumsOf is the digests the SHA256SUMS f holds, by file name,
// from the cache or the bucket; nil when it cannot be read, which leaves
// the manifest unchecked rather than marked.
func releaseSumsOf(ctx context.Context, bucket s3.Bucket, repo string, f uv.File) map[string]string {
return releaseFileOf(ctx, bucket, repo, f, "s", func(raw []byte) manifestEntry {
return manifestEntry{sums: parseSums(raw)}
}).sums
}
// releaseFileOf reads one of a release folder's own small files, f,
// through the cache: parse turns its bytes into an entry. kind keeps the
// two readers' entries for one file apart.
func releaseFileOf(ctx context.Context, bucket s3.Bucket, repo string, f uv.File, kind string, parse func([]byte) manifestEntry) manifestEntry {
base := f.Name[strings.LastIndexByte(f.Name, '/')+1:]
if f.Size > releaseManifestMax {
return manifestEntry{err: fmt.Sprintf("%s is %d bytes, more than this page reads", base, f.Size)}
}
key := f.Hash + "\x00" + kind
manifestCache.Lock()
e, ok := manifestCache.entries[key]
manifestCache.Unlock()
if ok {
return e
}
raw, _, err := bucket.Get(ctx, attachClient(), uv.Key(repo, f.Name), time.Now())
if err != nil {
// Not cached: the bucket may answer next time.
log.Printf("uv: %s: reading %s: %v", repo, f.Name, err)
return manifestEntry{err: base + " could not be read from the bucket"}
}
if !uv.Verify(f.Hash, raw) {
// Cached under the hash the catalog claims, which these bytes
// will never have: a rebuild that fixes the catalog changes
// the key.
e = manifestEntry{err: base + " in the bucket is not the file the catalog names; an admin can rebuild the catalog"}
} else {
e = parse(raw)
}
manifestCache.Lock()
if _, had := manifestCache.entries[key]; !had {
if len(manifestCache.order) >= manifestCacheEntries {
delete(manifestCache.entries, manifestCache.order[0])
manifestCache.order = manifestCache.order[1:]
}
manifestCache.entries[key] = e
manifestCache.order = append(manifestCache.order, key)
}
manifestCache.Unlock()
return e
}
// releaseVersion matches a folder name that reads as a version: dotted
// numbers after an optional v or version-, then an optional pre-release
// or build suffix.
var releaseVersion = regexp.MustCompile(`^(?:v|version-)?(\d+(?:\.\d+)*)([-+].*)?$`)
// compareVersions orders two names releaseVersion matched: numbers by
// value, a missing number as zero, a pre-release below its release.
func compareVersions(a, b []string) int {
na, nb := strings.Split(a[1], "."), strings.Split(b[1], ".")
for i := 0; i < max(len(na), len(nb)); i++ {
var x, y int
if i < len(na) {
x, _ = strconv.Atoi(na[i])
}
if i < len(nb) {
y, _ = strconv.Atoi(nb[i])
}
if x != y {
return x - y
}
}
pa, pb := strings.HasPrefix(a[2], "-"), strings.HasPrefix(b[2], "-")
switch {
case pa && !pb:
return -1
case pb && !pa:
return 1
}
return strings.Compare(a[2], b[2])
}
// orderReleases puts the newest first: by version when every name is
// one, else by the newest file in each group.
func orderReleases(rs []release) {
versions := make([][]string, len(rs))
allVersions := true
for i, r := range rs {
versions[i] = releaseVersion.FindStringSubmatch(r.Name)
allVersions = allVersions && versions[i] != nil
}
idx := make([]int, len(rs))
for i := range idx {
idx[i] = i
}
sort.SliceStable(idx, func(i, j int) bool {
a, b := idx[i], idx[j]
if allVersions {
if c := compareVersions(versions[a], versions[b]); c != 0 {
return c > 0
}
}
if rs[a].Mtime != rs[b].Mtime {
return rs[a].Mtime > rs[b].Mtime
}
return rs[a].Name > rs[b].Name
})
sorted := make([]release, len(rs))
for i, k := range idx {
sorted[i] = rs[k]
}
copy(rs, sorted)
for i := range rs {
rs[i].Newest = i == 0
}
}
// groupReleases is every live file under releases/<name>/, by folder.
func groupReleases(files []uv.File) []release {
byName := map[string]*release{}
var out []*release
for _, f := range files {
rest, ok := strings.CutPrefix(f.Name, releasesPrefix)
folder, file, nested := strings.Cut(rest, "/")
if !ok || !nested || folder == "" || file == "" || f.Deleted {
continue
}
r := byName[folder]
if r == nil {
r = &release{Name: folder}
byName[folder] = r
out = append(out, r)
}
r.Files = append(r.Files, f)
r.Mtime = max(r.Mtime, f.Mtime)
if file == releaseSumsName {
r.Sums = f.Name
}
}
rs := make([]release, len(out))
for i, r := range out {
sort.Slice(r.Files, func(a, b int) bool { return r.Files[a].Name < r.Files[b].Name })
rs[i] = *r
}
orderReleases(rs)
return rs
}
// changelogSection is the part of a CHANGELOG.md under the "## " heading
// that names release as a word of its own (a leading v allowed), up to
// the next heading of that level. body leaves the heading out.
func changelogSection(md []byte, name string) (heading, body string, ok bool) {
lines := strings.Split(string(md), "\n")
start := -1
for i, line := range lines {
h, isHeading := strings.CutPrefix(line, "## ")
if !isHeading {
continue
}
if start >= 0 {
return heading, strings.TrimSpace(strings.Join(lines[start+1:i], "\n")), true
}
for _, word := range strings.FieldsFunc(h, func(r rune) bool {
return r == ' ' || r == '\t' || strings.ContainsRune("[]()`*_,:;", r)
}) {
if word == name || word == "v"+name || word == "version-"+name {
heading, start = strings.TrimSpace(h), i
break
}
}
}
if start < 0 {
return "", "", false
}
return heading, strings.TrimSpace(strings.Join(lines[start+1:], "\n")), true
}
// repoChangelog is CHANGELOG.md at the tip of the default branch; nil
// when the repository has no store or no such file.
func (d Deps) repoChangelog(ctx context.Context, repo *Repo) []byte {
if repo.Zit == nil {
return nil
}
branch, _ := defaultBranchOf(repo)
if branch == "" {
branch = "trunk"
}
md, err := repo.Zit.Cat(ctx, "CHANGELOG.md", d.resolveCheckin(ctx, repo, branch))
if err != nil {
return nil
}
return md
}
// bucketBound is a BucketStore that can say whether a binding exists
// without unsealing its secret, as oltp's does.
type bucketBound interface {
BucketBound(repo string) (bool, error)
}
// uvBound is uvBinding's answer to "does this repository have
// unversioned files at all", from the database alone: no bucket call,
// and no secret unsealed where the store can say so without one. The
// pages asked on every render (the bar, the Files box) must agree with
// release.list, which has none without a bucket of the repository's own.
func (d Deps) uvBound(repo string) bool {
if d.UV == nil || d.Buckets == nil || d.companionParent(repo) != "" {
return false
}
var bound bool
var err error
if bb, ok := d.Buckets.(bucketBound); ok {
bound, err = bb.BucketBound(repo)
} else {
_, bound, err = d.Buckets.Bucket(repo)
}
if err != nil {
log.Printf("uv: %s: binding: %v", repo, err)
}
return bound
}
// hasReleases is the repository bar's question, asked on every page,
// so it is answered by existence from the catalog and never lists a
// bucket.
func (d Deps) hasReleases(repo string) bool {
if !d.uvBound(repo) {
return false
}
has, err := d.UV.UVHasReleases(repo)
if err != nil {
log.Printf("uv: %s: catalog: %v", repo, err)
}
return has
}
func registerReleases(r *ops.Registry, d Deps) error {
if err := ops.Register(r, ops.Op[releaseSummaryIn, releaseSummaryOut]{
Name: "release.summary",
Caps: "o",
Doc: "How many releases a repository has and which is newest, ordered as release.list orders them, from the catalog alone: no bucket read, no manifest. Zero when it has none or no bucket.",
Kind: ops.Query,
Auth: ops.Anonymous,
Route: "GET /api/v1/r/{repo}/releases/summary",
Run: func(ctx context.Context, in releaseSummaryIn) (releaseSummaryOut, error) {
if _, err := d.repoFor(ctx, in.Repo); err != nil {
return releaseSummaryOut{}, err
}
if !d.uvBound(in.Repo) {
return releaseSummaryOut{}, nil
}
folders, err := d.UV.UVReleaseFolders(in.Repo)
if err != nil {
return releaseSummaryOut{}, err
}
if len(folders) == 0 {
return releaseSummaryOut{}, nil
}
rs := make([]release, len(folders))
for i, f := range folders {
rs[i] = release{Name: f.Name, Mtime: f.Mtime}
}
orderReleases(rs)
return releaseSummaryOut{Count: len(rs), Latest: rs[0].Name, Mtime: rs[0].Mtime}, nil
},
}); err != nil {
return err
}
return ops.Register(r, ops.Op[releaseListIn, releaseListOut]{
Name: "release.list",
Caps: "o",
Doc: "A repository's releases (RFC 0020), or with `name` the one release in that folder: its unversioned files under releases/<name>/, one group per folder, newest first (by version when every folder name is one, else by the newest file), the first marked newest. A group's release-manifest.json is read in zit's schema version 1 (artifacts with target, size and sha256; the source check-in; the date; omitted targets; qualification notes); a group's SHA256SUMS is named, and an artifact whose manifest sha256 differs from it carries SHA256SUMS's digest as sumsSha256; the CHANGELOG.md section whose heading names the release is rendered beside it.",
Kind: ops.Query,
Auth: ops.Anonymous,
Route: "GET /api/v1/r/{repo}/releases",
Run: func(ctx context.Context, in releaseListIn) (releaseListOut, error) {
repo, err := d.repoFor(ctx, in.Repo)
if err != nil {
return releaseListOut{}, err
}
if in.Name != "" {
if err := checkReleaseName(in.Name); err != nil {
return releaseListOut{}, err
}
}
bucket, _, bound, err := d.uvReady(ctx, in.Repo)
if err != nil {
return releaseListOut{}, err
}
if !bound {
return releaseListOut{Releases: []release{}, Note: NoUVBucket}, nil
}
files, err := d.UV.UVReleaseFiles(in.Repo)
if err != nil {
return releaseListOut{}, err
}
out := releaseListOut{Bound: true, Releases: groupReleases(files)}
if in.Name != "" {
// Grouped and ordered whole first, so the one asked for
// still says whether it is the newest.
var one []release
for _, rel := range out.Releases {
if rel.Name == in.Name {
one = append(one, rel)
}
}
if one == nil {
return releaseListOut{}, fmt.Errorf("no release %q: %w", in.Name, ops.ErrNotFound)
}
out.Releases = one
}
var changelog []byte
if len(out.Releases) > 0 {
changelog = d.repoChangelog(ctx, repo)
}
for i := range out.Releases {
rel := &out.Releases[i]
group := map[string]string{}
var manifest, sumsFile *uv.File
for k, f := range rel.Files {
base := strings.TrimPrefix(f.Name, releasesPrefix+rel.Name+"/")
group[base] = f.Name
switch base {
case releaseManifestName:
manifest = &rel.Files[k]
case releaseSumsName:
sumsFile = &rel.Files[k]
}
}
if manifest != nil {
m, errText := releaseManifestOf(ctx, bucket, in.Repo, rel.Name, *manifest)
if m != nil {
var sums map[string]string
if sumsFile != nil {
sums = releaseSumsOf(ctx, bucket, in.Repo, *sumsFile)
}
rel.Manifest = m.withFiles(group, sums)
}
rel.ManifestErr = errText
}
if heading, body, ok := changelogSection(changelog, rel.Name); ok {
html, rerr := renderBodyWith(nil, "text/x-markdown", []byte(body))
if rerr == nil {
rel.Notes = &releaseNotes{Heading: heading, HTML: html}
}
}
}
return out, nil
},
})
}