// uv.go is RFC 0020 rung 2: a repository's unversioned files, fossil's
// second kind of content, kept in the repository's own bucket beside
// its attachments and catalogued in OLTP.
//
// The bucket is the record. Each file's content is the object
// unversioned/<name>, with fossil's mtime and content hash (and who
// wrote it) as the object's metadata; a removal is an empty object at
// unversioned/.tombstones/<name> carrying the removal's mtime. The
// catalog is a copy, so a listing and the sync front's uv-hash never
// list the bucket on a request path, and it is rebuilt from a listing
// whenever it was never built or a different bucket is bound.
//
// No bucket, no unversioned files: the forge holds none of the bytes
// and owns no storage for them, as RFC 0016 owns none for attachments.
package api
import (
"bytes"
"context"
"encoding/base64"
"errors"
"fmt"
"io"
"log"
"net/http"
"net/url"
"path"
"strconv"
"strings"
"sync"
"time"
"craftdesign.group/quarry/internal/evt"
"craftdesign.group/quarry/internal/oltp"
"craftdesign.group/quarry/internal/ops"
"craftdesign.group/quarry/internal/s3"
"craftdesign.group/quarry/internal/uv"
)
// UVStore is the catalog (internal/oltp/unversioned.go). nil is a forge
// with no OLTP, which has no unversioned files.
type UVStore interface {
UVFiles(repo string) ([]uv.File, error)
UVFile(repo, name string) (uv.File, bool, error)
PutUVFile(repo string, f uv.File) (bool, error)
UVSource(repo string) (string, bool, error)
ReplaceUVCatalog(repo, source string, files []uv.File) error
DropUVCatalog(repo string) error
// UVHasReleases is whether any live name sits in a folder under
// releases/, by existence; UVReleaseFiles is those names.
UVHasReleases(repo string) (bool, error)
UVReleaseFiles(repo string) ([]uv.File, error)
// UVReleaseFolders is one row per release folder with its newest
// mtime.
UVReleaseFolders(repo string) ([]oltp.UVFolder, error)
}
// NoUVBucket is what a repository without a bucket of its own says
// about unversioned files, on the listing and on every refused write.
const NoUVBucket = "bind a bucket in Settings to publish unversioned files"
// Metadata names on the objects. S3 lower-cases them and so do we.
const (
uvMetaMtime = "mtime"
uvMetaHash = "hash"
uvMetaWho = "who"
// uvMetaName is the name, percent-encoded, so a rebuild reads it
// back exactly whatever a listing does to the key.
uvMetaName = "uv-name"
)
// uvLocks serialises writes and rebuilds per repository. Without it a
// rebuild that listed the bucket before a put wrote its object would
// replace the catalog after the put recorded its row, and the row
// would be lost with the catalog still naming the bucket as built.
// Unversioned writes are rare and a release is a handful of files, so
// one lock per repository costs nothing that matters.
//
// The lock is this process's. It is enough because one forge process
// owns an OLTP database (the writer connection, oltp.Open); two
// processes over one database would need it moved into the database,
// and running.md says so. It is not in OLTP today because what it
// guards spans a bucket request, which a SQLite transaction should not
// be held open across.
var uvLocks sync.Map
func uvLock(repo string) func() {
m, _ := uvLocks.LoadOrStore(repo, &sync.Mutex{})
mu := m.(*sync.Mutex)
mu.Lock()
return mu.Unlock
}
// uvPublicRefusal refuses unversioned writes for a private repository
// whose bucket serves a public origin. An attachment's key is its
// content hash, which nobody can guess; an unversioned file's key is its
// name, unversioned/<repo>/download.html, which anybody can, so a public base
// would hand a private repository's files to whoever asks for them by
// name. Reads already never link the public base for a private
// repository; this keeps the bytes from getting there at all.
func (d Deps) uvPublicRefusal(repo string, b oltp.BucketBinding) error {
if b.PublicBase == "" || d.Private == nil || !d.Private()[repo] {
return nil
}
return fmt.Errorf("%s is private and its bucket is served publicly from %s, where an unversioned file "+
"is readable by anyone who guesses its name: remove the public base in Settings to publish unversioned files: %w",
repo, b.PublicBase, ops.ErrInvalid)
}
// uvBinding is a repository's bucket for unversioned files. ok is false
// when it has none of its own; a companion store has none either, since
// its parent's bucket holds the parent's files under the parent's prefix.
func (d Deps) uvBinding(repo string) (s3.Bucket, oltp.BucketBinding, BucketInfo, bool, error) {
if d.UV == nil || d.Buckets == nil || d.companionParent(repo) != "" {
return s3.Bucket{}, oltp.BucketBinding{}, BucketInfo{}, false, nil
}
info, b, bucket, err := d.boundBucket(repo)
if errors.Is(err, ops.ErrInvalid) {
return s3.Bucket{}, oltp.BucketBinding{}, BucketInfo{}, false, nil
}
if err != nil {
return s3.Bucket{}, oltp.BucketBinding{}, BucketInfo{}, false, err
}
return bucket, b, info, true, nil
}
// uvNameOf is the file an object holds. The name the object carries in
// its own metadata wins over the one read back from the listing's key:
// a key comes back through the service's key encoding, which S3 defines
// and other services may not follow to the letter, while the metadata
// is exactly what the forge wrote. Objects written before the name was
// recorded fall back to the key. Whether it is a removal is the key's
// to say, since the tombstone prefix is plain ASCII. key is one of
// repo's, from a listing under its prefix.
func uvNameOf(repo, key, meta string) (name string, tombstone, ok bool) {
tombstone = uv.IsTombstoneKey(repo, key)
if n, err := url.QueryUnescape(meta); meta != "" && err == nil && uv.CheckName(n) == nil {
return n, tombstone, true
}
return uv.NameOfKey(repo, key)
}
// uvSource names the bucket a catalog was built from.
func uvSource(b oltp.BucketBinding) string { return b.Endpoint + "/" + b.Bucket }
// uvEnsure rebuilds repo's catalog when it was never built or was built
// from another bucket. The caller holds uvLock(repo).
func (d Deps) uvEnsure(ctx context.Context, repo string, bucket s3.Bucket, b oltp.BucketBinding) error {
src, ok, err := d.UV.UVSource(repo)
if err != nil {
return err
}
if ok && src == uvSource(b) {
return nil
}
_, err = d.uvRebuild(ctx, repo, bucket, b)
return err
}
// uvRebuild reads the catalog back from the bucket: every object under
// repo's own prefix, unversioned/<repo>/, so another repository bound
// to the same bucket is never imported; its metadata by HEAD, content
// and tombstone of one name reconciled by fossil's rule. An object without a readable mtime is
// not one this layout wrote and is skipped. The caller holds
// uvLock(repo).
func (d Deps) uvRebuild(ctx context.Context, repo string, bucket s3.Bucket, b oltp.BucketBinding) ([]uv.File, error) {
client := attachClient()
now := time.Now()
prefix := uv.RepoPrefix(repo)
objects, err := bucket.List(ctx, client, prefix, now)
if err != nil {
return nil, fmt.Errorf("listing unversioned files in the bucket: %w", err)
}
byName := map[string]uv.File{}
for _, o := range objects {
if !strings.HasPrefix(o.Key, prefix) {
continue
}
obj, found, err := bucket.Stat(ctx, client, o.Key, now)
if err != nil {
return nil, fmt.Errorf("reading %s: %w", o.Key, err)
}
if !found {
continue // removed between the listing and now
}
name, tombstone, ok := uvNameOf(repo, o.Key, obj.Meta[uvMetaName])
if !ok {
log.Printf("uv: %s: %s names no unversioned file; skipped", repo, o.Key)
continue
}
mtime, err := strconv.ParseInt(obj.Meta[uvMetaMtime], 10, 64)
if err != nil {
log.Printf("uv: %s: %s carries no mtime; skipped", repo, o.Key)
continue
}
who, _ := url.QueryUnescape(obj.Meta[uvMetaWho])
f := uv.File{Name: name, Mtime: mtime, Who: who, Deleted: tombstone}
if !tombstone {
f.Hash, f.Size = obj.Meta[uvMetaHash], obj.Size
if f.Hash == "" {
log.Printf("uv: %s: %s carries no hash; skipped", repo, o.Key)
continue
}
}
// A tombstone alone is still a row: unlike a sync, a rebuild is
// reading back what this forge recorded, removals included.
if have, exists := byName[name]; !exists || uv.Replaces(f, have, true) {
byName[name] = f
}
}
files := make([]uv.File, 0, len(byName))
for _, f := range byName {
files = append(files, f)
}
if err := d.UV.ReplaceUVCatalog(repo, uvSource(b), files); err != nil {
return nil, err
}
return d.UV.UVFiles(repo)
}
// uvReady resolves a repository's binding once and makes sure its
// catalog is built from that bucket. bound is false for a repository
// with no bucket of its own.
func (d Deps) uvReady(ctx context.Context, repo string) (s3.Bucket, BucketInfo, bool, error) {
bucket, b, info, ok, err := d.uvBinding(repo)
if err != nil || !ok {
return s3.Bucket{}, BucketInfo{}, false, err
}
src, built, err := d.UV.UVSource(repo)
if err != nil {
return s3.Bucket{}, BucketInfo{}, true, err
}
if !built || src != uvSource(b) {
unlock := uvLock(repo)
err := d.uvEnsure(ctx, repo, bucket, b)
unlock()
if err != nil {
return s3.Bucket{}, BucketInfo{}, true, err
}
}
return bucket, info, true, nil
}
// UVCatalog is a repository's unversioned files, tombstones included,
// built from the bucket first when it needs to be. bound is false for a
// repository with no bucket of its own, which has none. The sync front
// answers pragma uv-hash and uvigot from this.
func (d Deps) UVCatalog(ctx context.Context, repo string) (files []uv.File, bound bool, err error) {
if _, _, bound, err = d.uvReady(ctx, repo); err != nil || !bound {
return []uv.File{}, bound, err
}
files, err = d.UV.UVFiles(repo)
return files, true, err
}
// UVWrite applies one version of a file, content or tombstone (f.Deleted),
// by fossil's rule, and reports whether it applied. f.Hash, when set, is
// verified against content and kept as given, so a SHA1 repository's
// hash stays SHA1; when empty it is computed (uv.Hash). The object is
// written before the row, so a row never names bytes the bucket lacks.
//
// It checks the name, the binding's size limit and the hash; the caller
// checks who may write and whether the repository takes local writes,
// because a mirror's import writes into a store that refuses its users.
// A version that loses to what the catalog holds is not applied and is
// not an error: fossil's receiver drops it the same way.
func (d Deps) UVWrite(ctx context.Context, repo string, f uv.File, content []byte) (uv.File, bool, error) {
if err := uv.CheckName(f.Name); err != nil {
return uv.File{}, false, fmt.Errorf("%v: %w", err, ops.ErrInvalid)
}
if f.Mtime < 0 {
return uv.File{}, false, fmt.Errorf("an mtime is seconds since 1970 and not negative: %w", ops.ErrInvalid)
}
bucket, b, _, ok, err := d.uvBinding(repo)
if err != nil {
return uv.File{}, false, err
}
if !ok {
return uv.File{}, false, fmt.Errorf("%s has no bucket of its own: %s: %w", repo, NoUVBucket, ops.ErrInvalid)
}
if err := d.uvPublicRefusal(repo, b); err != nil {
return uv.File{}, false, err
}
if f.Deleted {
f.Hash, f.Size, content = "", 0, nil
} else {
if int64(len(content)) > b.LimitBytes {
return uv.File{}, false, fmt.Errorf("%s is %d bytes, over this bucket's limit of %d: %w", f.Name, len(content), b.LimitBytes, ops.ErrInvalid)
}
switch {
case f.Hash == "":
f.Hash = uv.Hash(content)
case !uv.Verify(f.Hash, content):
return uv.File{}, false, fmt.Errorf("the hash given for %s is not its content's: %w", f.Name, ops.ErrInvalid)
}
f.Size = int64(len(content))
}
unlock := uvLock(repo)
defer unlock()
if err := d.uvEnsure(ctx, repo, bucket, b); err != nil {
return uv.File{}, false, err
}
have, exists, err := d.UV.UVFile(repo, f.Name)
if err != nil {
return uv.File{}, false, err
}
if !uv.Replaces(f, have, exists) {
return have, false, nil
}
client, now := uvClient(int64(len(content))), time.Now()
opt := uvPutOptions(f)
key := uv.TombstoneKey(repo, f.Name)
if !f.Deleted {
key = uv.Key(repo, f.Name)
}
if err := bucket.PutWith(ctx, client, key, content, opt, now); err != nil {
return uv.File{}, false, fmt.Errorf("writing %s to the bucket: %w", key, err)
}
if err := d.uvRecord(repo, &f, now); err != nil {
return uv.File{}, false, err
}
// The other half of the pair goes, so the bucket holds one object
// per name. Left behind it would still be harmless, since a rebuild
// takes the newer of the two; that is why a failure here is logged
// and not returned.
stale := ""
switch {
case f.Deleted:
stale = uv.Key(repo, f.Name)
case exists && have.Deleted:
stale = uv.TombstoneKey(repo, f.Name)
}
if stale != "" {
if err := bucket.Delete(ctx, client, stale, now); err != nil {
log.Printf("uv: %s: removing %s: %v", repo, stale, err)
}
}
d.raiseUVChanged(ctx, repo, f)
return f, true, nil
}
// uvPutOptions is the object a version is stored as: its mtime, who
// wrote it and its name as metadata, and for content its hash and how
// a download is served.
func uvPutOptions(f uv.File) s3.PutOptions {
opt := s3.PutOptions{ContentType: "application/octet-stream",
Meta: map[string]string{uvMetaMtime: strconv.FormatInt(f.Mtime, 10), uvMetaWho: url.QueryEscape(f.Who),
uvMetaName: url.QueryEscape(f.Name)}}
if f.Deleted {
return opt
}
opt.Meta[uvMetaHash] = f.Hash
// The object carries the same answer uv.get signs into its URL,
// because a bucket with a public origin serves the object as
// stored and nothing signs that request: anything uv.get would not
// show in place is stored as bytes to download.
if ctype, inline := uv.ContentType(f.Name); inline {
opt.ContentType = ctype
} else {
opt.Disposition = "attachment; filename*=UTF-8''" + url.PathEscape(path.Base(f.Name))
}
return opt
}
// uvRecord puts the row for a version whose object is already in the
// bucket. The caller holds uvLock(repo).
func (d Deps) uvRecord(repo string, f *uv.File, now time.Time) error {
f.At = now.UTC().Format(time.RFC3339)
applied, err := d.UV.PutUVFile(repo, *f)
if err == nil && applied {
return nil
}
// The bucket now holds bytes the catalog does not describe. The
// bucket is the record, so the copy goes and the next read rebuilds
// it from the bucket; if even the drop fails, the catalog is wrong
// until uv.rebuild, and the log says so.
if derr := d.UV.DropUVCatalog(repo); derr != nil {
log.Printf("uv: %s: catalog disagrees with the bucket over %s and could not be dropped: %v", repo, f.Name, derr)
}
if err == nil {
err = fmt.Errorf("%s changed while it was being written; the catalog will be rebuilt from the bucket", f.Name)
}
return err
}
// raiseUVChanged is RFC 0005's notice of a write. Operational, like
// mirror.*: the catalog row is the record and this tells subscribers
// about it.
func (d Deps) raiseUVChanged(ctx context.Context, repo string, f uv.File) {
if d.Events == nil {
return
}
if err := d.Events.Raise(evt.UVChanged, repo, map[string]any{
"repo": repo, "name": f.Name, "deleted": f.Deleted, "actor": f.Who,
}); err != nil {
log.Printf("events: raise %s: %v", evt.UVChanged, err)
}
}
type uvListIn struct {
Repo string `json:"repo" validate:"required,max=64"`
}
type uvListOut struct {
// Files is every name with its newest version, tombstones included
// (deleted true, hash empty), by name.
Files []uv.File `json:"files"`
// Bound is false for a repository with no bucket of its own, which
// has no unversioned files; Note then says what to do.
Bound bool `json:"bound"`
Note string `json:"note,omitempty"`
// Limit is the bucket's per-file limit in bytes, what an upload
// may be; zero when unbound.
Limit int64 `json:"limit,omitempty"`
// Hash is fossil's uv-hash over Files, the value a client compares
// with its own to know whether anything differs.
Hash string `json:"hash"`
}
type uvGetIn struct {
Repo string `json:"repo" validate:"required,max=64"`
Name string `json:"name" validate:"required,max=500"`
}
type uvGetOut struct {
File uv.File `json:"file"`
// URL is where the bytes are: a presigned GET good for an hour, or
// the bucket's public base for a public repository's passive file.
URL string `json:"url"`
Expires string `json:"expires,omitempty"`
Type string `json:"type"`
// Inline says the URL shows the file in place; anything that could
// run in a browser is always a download.
Inline bool `json:"inline"`
}
type uvPutIn struct {
Repo string `json:"repo" validate:"required,max=64"`
Name string `json:"name" validate:"required,max=500"`
// Mtime is fossil's, seconds since 1970; zero means now.
Mtime int64 `json:"mtime" validate:"min=0"`
// Content is the file, base64. It crosses the op boundary as text
// because an op's input is JSON; the bound on it is the binding's
// per-file limit, checked on the decoded bytes.
Content string `json:"content"`
// Hash, when given, must be the content's under SHA1 or SHA3-256
// and is kept as given; empty computes SHA3-256.
Hash string `json:"hash" validate:"max=64"`
// raw is the content as bytes, set only by UVUploadForm in this
// package: the form path goes through the op, gates and all, without
// a base64 copy of the file. JSON and form decoding cannot reach it.
raw []byte
}
type uvPutOut struct {
File uv.File `json:"file"`
// Applied is false when the catalog already held exactly this
// version: nothing was written and nothing was raised.
Applied bool `json:"applied"`
}
type uvRmIn struct {
Repo string `json:"repo" validate:"required,max=64"`
Name string `json:"name" validate:"required,max=500"`
// Mtime is the removal's; zero means now.
Mtime int64 `json:"mtime" validate:"min=0"`
// IfMtime, when given, is the version the remover saw: the removal
// is refused if the file has changed since, and is dated after that
// version (now, or its mtime plus one, whichever is later) unless
// Mtime says otherwise.
IfMtime int64 `json:"ifMtime" validate:"min=0"`
}
type uvRebuildIn struct {
Repo string `json:"repo" validate:"required,max=64"`
}
// uvBodyLimit is the most of a uv.put request worth reading for a
// binding whose per-file limit is limit: the content in base64, four
// bytes for every three, and room for the name, the hash and JSON.
func uvBodyLimit(limit int64) int64 {
return limit/3*4 + 4 + 16<<10
}
// UVFormLimit is the most the page's upload form carries through the
// forge, whatever the binding allows: the file is held in memory once
// on its way to the bucket. Larger files publish with fossil uv.
const UVFormLimit = maxFormUploadBytes
// uvFormSlots bounds how many form uploads hold a file in memory at
// once.
var uvFormSlots = make(chan struct{}, 4)
// UVUploadForm is the page's upload: a multipart form with `file` and
// the optional `prefix` and `name`, streamed (never spilled to disk)
// into one buffer bounded by the binding's limit and UVFormLimit, then
// published through uv.put, so every gate is the op's. It returns the
// name published. ctx must carry the caller as a gated request does.
func UVUploadForm(ctx context.Context, d Deps, r *ops.Registry, repo string, w http.ResponseWriter, req *http.Request) (string, error) {
select {
case uvFormSlots <- struct{}{}:
defer func() { <-uvFormSlots }()
default:
return "", fmt.Errorf("too many uploads are in progress; try again in a moment: %w", ops.ErrUnavailable)
}
_, b, _, bound, err := d.uvBinding(repo)
if err != nil {
return "", err
}
if !bound {
return "", fmt.Errorf("%s has no bucket of its own: %s: %w", repo, NoUVBucket, ops.ErrInvalid)
}
limit := min(b.LimitBytes, UVFormLimit)
// The whole body is bounded, not only the file part, and a client
// that sends slowly loses its slot: a minute, plus a second for every
// 256 KiB the limit allows.
req.Body = http.MaxBytesReader(w, req.Body, limit+64<<10)
if err := http.NewResponseController(w).SetReadDeadline(time.Now().Add(time.Minute + time.Duration(limit/(256<<10))*time.Second)); err != nil && !errors.Is(err, http.ErrNotSupported) {
log.Printf("uv: %s: upload deadline: %v", repo, err)
}
mr, err := req.MultipartReader()
if err != nil {
return "", fmt.Errorf("expected a multipart form with a file: %w", ops.ErrInvalid)
}
var body bytes.Buffer
var prefix, name, filename string
got := false
for {
part, err := mr.NextPart()
if errors.Is(err, io.EOF) {
break
}
if err != nil {
return "", fmt.Errorf("%v: %w", err, ops.ErrInvalid)
}
switch part.FormName() {
case "prefix", "name":
text, _ := io.ReadAll(io.LimitReader(part, uv.MaxName+1))
if part.FormName() == "prefix" {
prefix = string(text)
} else {
name = string(text)
}
case "file":
filename = part.FileName()
n, err := io.Copy(&body, io.LimitReader(part, limit+1))
if err != nil {
return "", err
}
if n > limit {
if limit < b.LimitBytes {
return "", fmt.Errorf("files over %d MB publish with fossil uv add and fossil sync -u, not through this form: %w", limit>>20, ops.ErrInvalid)
}
return "", fmt.Errorf("the file is over this bucket's limit of %d bytes: %w", b.LimitBytes, ops.ErrInvalid)
}
got = true
default:
io.Copy(io.Discard, part)
}
part.Close()
}
if !got {
return "", fmt.Errorf("choose a file to publish: %w", ops.ErrInvalid)
}
full := uvUploadName(prefix, name, filename)
if _, err := r.Invoke(ctx, "uv.put", uvPutIn{Repo: repo, Name: full, raw: body.Bytes()}); err != nil {
return "", err
}
return full, nil
}
// uvUploadName is the name an upload publishes under: the one given,
// else the file's own, under the folder given.
func uvUploadName(prefix, name, filename string) string {
if name = strings.TrimSpace(name); name == "" {
name = path.Base(strings.ReplaceAll(strings.TrimSpace(filename), `\`, "/"))
}
if prefix = strings.Trim(strings.TrimSpace(prefix), "/"); prefix != "" {
return prefix + "/" + name
}
return name
}
func uvMtime(m int64) int64 {
if m == 0 {
return time.Now().Unix()
}
return m
}
// uvWriteGate is what uv.put and uv.rm ask before UVWrite: the letter,
// and a store that takes local writes. A tracking mirror takes none,
// unversioned ones included (RFC 0020, the sync front).
func (d Deps) uvWriteGate(ctx context.Context, repo string) error {
if _, err := d.repoFor(ctx, repo); err != nil {
return err
}
if !d.holds(ctx, repo, "y") {
return fmt.Errorf("publishing unversioned files needs y (Write-UV), which admin does not imply: %w", ops.ErrForbidden)
}
return d.refuseUnlessWritable(repo)
}
func (d Deps) uvRefused(repo string, f uv.File) error {
if f.Deleted {
return fmt.Errorf("%s already holds %s at mtime %d, at least as new as this: %w", repo, f.Name, f.Mtime, ops.ErrInvalid)
}
return fmt.Errorf("%s already holds %s at mtime %d with hash %s, at least as new as this: %w", repo, f.Name, f.Mtime, f.Hash, ops.ErrInvalid)
}
// registerUV mounts uv.list, uv.get, uv.put, uv.rm and uv.rebuild.
// Registered on every forge, like the bucket ops; one without a
// catalog lists nothing and refuses writes in the same words as a
// repository with no bucket.
func registerUV(r *ops.Registry, d Deps) error {
if err := ops.Register(r, ops.Op[uvListIn, uvListOut]{
Name: "uv.list",
Caps: "o",
Doc: "A repository's unversioned files (RFC 0020): each name's newest version with its fossil mtime, hash, size and who wrote it, removals included as `deleted` rows, and fossil's uv-hash over them. A repository with no bucket of its own has none: `bound` is false and `note` says how to get one.",
Kind: ops.Query,
Auth: ops.Anonymous,
Route: "GET /api/v1/r/{repo}/uv",
Run: func(ctx context.Context, in uvListIn) (uvListOut, error) {
if _, err := d.repoFor(ctx, in.Repo); err != nil {
return uvListOut{}, err
}
files, bound, err := d.UVCatalog(ctx, in.Repo)
if err != nil {
return uvListOut{}, err
}
out := uvListOut{Files: files, Bound: bound, Hash: uv.CatalogHash(files)}
if !bound {
out.Note = NoUVBucket
} else if _, b, _, ok, err := d.uvBinding(in.Repo); err == nil && ok {
out.Limit = b.LimitBytes
}
return out, nil
},
}); err != nil {
return err
}
if err := ops.Register(r, ops.Op[uvGetIn, uvGetOut]{
Name: "uv.get",
Caps: "o",
Doc: "Where one unversioned file's bytes are: a presigned GET on the repository's bucket good for an hour, or its public base for a public repository's passive file. The type comes from the name's extension by a fixed table; HTML, SVG, script and anything else that could run in a browser is served as a download, never in place.",
Kind: ops.Query,
Auth: ops.Anonymous,
Route: "GET /api/v1/r/{repo}/uv/{name...}",
Run: func(ctx context.Context, in uvGetIn) (uvGetOut, error) {
if _, err := d.repoFor(ctx, in.Repo); err != nil {
return uvGetOut{}, err
}
bucket, info, bound, err := d.uvReady(ctx, in.Repo)
if err != nil {
return uvGetOut{}, err
}
if !bound {
return uvGetOut{}, fmt.Errorf("no unversioned file %q; %s: %w", in.Name, NoUVBucket, ops.ErrNotFound)
}
f, ok, err := d.UV.UVFile(in.Repo, in.Name)
if err != nil {
return uvGetOut{}, err
}
if !ok || f.Deleted {
return uvGetOut{}, fmt.Errorf("no unversioned file %q: %w", in.Name, ops.ErrNotFound)
}
ctype, inline := uv.ContentType(f.Name)
out := uvGetOut{File: f, Type: ctype, Inline: inline}
private := d.Private != nil && d.Private()[in.Repo]
// A public base cannot carry a disposition, so only a file
// that may render in place is linked there.
if info.PublicBase != "" && !private && inline {
out.URL = info.PublicBase + "/" + (&url.URL{Path: uv.Key(in.Repo, f.Name)}).EscapedPath()
return out, nil
}
now := time.Now()
u, err := bucket.PresignGet(uv.Key(in.Repo, f.Name), linkExpiry, now, path.Base(f.Name), ctype, inline)
if err != nil {
return uvGetOut{}, err
}
out.URL, out.Expires = u.String(), now.Add(linkExpiry).UTC().Format(time.RFC3339)
return out, nil
},
}); err != nil {
return err
}
if err := ops.Register(r, ops.Op[uvPutIn, uvPutOut]{
Name: "uv.put",
Caps: "y",
Doc: "Publish an unversioned file, replacing the name's current version when this one is newer by fossil's rule (the larger mtime, then the larger hash); the version already held, same mtime and hash, is nothing to do and answers `applied` false. Fossil's name rules apply, the size is bounded by the bucket's per-file limit, and the object is written to the repository's bucket before the catalog records it with who wrote it. Needs y, which admin does not imply; refused on a mirror that takes no local writes. Raises uv.changed.",
Kind: ops.Command,
Auth: ops.Token,
Route: "POST /api/v1/r/{repo}/uv",
MaxBody: func(repo string) int64 {
_, b, _, ok, err := d.uvBinding(repo)
if err != nil || !ok {
// Refused anyway; read no more than it takes to say so.
return uvBodyLimit(0)
}
return uvBodyLimit(b.LimitBytes)
},
Run: func(ctx context.Context, in uvPutIn) (uvPutOut, error) {
if err := d.uvWriteGate(ctx, in.Repo); err != nil {
return uvPutOut{}, err
}
content := in.raw
if content == nil {
var err error
if content, err = base64.StdEncoding.DecodeString(in.Content); err != nil {
return uvPutOut{}, fmt.Errorf("content is not base64: %w", ops.ErrInvalid)
}
}
mtime := uvMtime(in.Mtime)
f, applied, err := d.UVWrite(ctx, in.Repo, uv.File{Name: in.Name, Mtime: mtime, Hash: in.Hash, Who: ops.User(ctx)}, content)
if err != nil {
return uvPutOut{}, err
}
if !applied {
// The same mtime and hash is fossil's "nothing to do",
// not a conflict: a retried publish succeeds.
if !f.Deleted && f.Mtime == mtime && uv.Verify(f.Hash, content) {
return uvPutOut{File: f}, nil
}
return uvPutOut{}, d.uvRefused(in.Repo, f)
}
return uvPutOut{File: f, Applied: true}, nil
},
}); err != nil {
return err
}
if err := ops.Register(r, ops.Op[uvRmIn, uvPutOut]{
Name: "uv.rm",
Caps: "y",
Doc: "Remove an unversioned file: a tombstone object in the bucket and a `deleted` row in the catalog, both carrying the removal's mtime, which is what syncs the removal. The removal must be newer than the file by fossil's rule, so a removal in the same second as the put it removes is refused (a tombstone loses a tie), as it is in fossil; pass a later mtime or wait a second. Needs y, which admin does not imply. Raises uv.changed.",
Kind: ops.Command,
Auth: ops.Token,
Route: "POST /api/v1/r/{repo}/uv/rm",
Run: func(ctx context.Context, in uvRmIn) (uvPutOut, error) {
if err := d.uvWriteGate(ctx, in.Repo); err != nil {
return uvPutOut{}, err
}
if err := uv.CheckName(in.Name); err != nil {
return uvPutOut{}, fmt.Errorf("%v: %w", err, ops.ErrInvalid)
}
_, _, bound, err := d.uvReady(ctx, in.Repo)
if err != nil {
return uvPutOut{}, err
}
if !bound {
return uvPutOut{}, fmt.Errorf("%s has no bucket of its own: %s: %w", in.Repo, NoUVBucket, ops.ErrInvalid)
}
have, held, err := d.UV.UVFile(in.Repo, in.Name)
if err != nil {
return uvPutOut{}, err
} else if !held || have.Deleted {
return uvPutOut{}, fmt.Errorf("no unversioned file %q: %w", in.Name, ops.ErrNotFound)
}
mtime := uvMtime(in.Mtime)
if in.IfMtime != 0 {
if have.Mtime != in.IfMtime {
return uvPutOut{}, fmt.Errorf("%s has been replaced since you saw it; look again before removing it: %w", in.Name, ops.ErrInvalid)
}
if in.Mtime == 0 {
mtime = max(mtime, have.Mtime+1)
}
}
f, applied, err := d.UVWrite(ctx, in.Repo, uv.File{Name: in.Name, Mtime: mtime, Deleted: true, Who: ops.User(ctx)}, nil)
if err != nil {
return uvPutOut{}, err
}
if !applied {
return uvPutOut{}, d.uvRefused(in.Repo, f)
}
return uvPutOut{File: f, Applied: true}, nil
},
}); err != nil {
return err
}
if err := ops.Register(r, ops.Op[uvRebuildIn, uvListOut]{
Name: "uv.rebuild",
Caps: "a",
Doc: "Rebuild the repository's unversioned catalog from its bucket: every object under unversioned/ and its metadata, removals included. The catalog rebuilds itself when it was never built or another bucket is bound; this is for a bucket changed by hand behind the forge's back.",
Kind: ops.Command,
Auth: ops.Token,
Route: "POST /api/v1/r/{repo}/uv/rebuild",
Run: func(ctx context.Context, in uvRebuildIn) (uvListOut, error) {
if _, err := d.repoFor(ctx, in.Repo); err != nil {
return uvListOut{}, err
}
if !d.holds(ctx, in.Repo, "a") {
return uvListOut{}, fmt.Errorf("rebuilding the unversioned catalog needs a: %w", ops.ErrForbidden)
}
bucket, b, _, ok, err := d.uvBinding(in.Repo)
if err != nil {
return uvListOut{}, err
}
if !ok {
return uvListOut{}, fmt.Errorf("%s has no bucket of its own: %s: %w", in.Repo, NoUVBucket, ops.ErrInvalid)
}
unlock := uvLock(in.Repo)
files, err := d.uvRebuild(ctx, in.Repo, bucket, b)
unlock()
if err != nil {
return uvListOut{}, err
}
return uvListOut{Files: files, Bound: true, Hash: uv.CatalogHash(files)}, nil
},
}); err != nil {
return err
}
return registerReleases(r, d)
}