// attachwrite.go is RFC 0016 Phase 3: writing an attachment through the
// forge. The browser never sends the file to quarry — begin mints a
// presigned PUT whose signature fixes the key, the length and the type;
// the browser PUTs; commit checks what landed and writes the stub
// through zit. What the forge holds between the two is a ticket, so
// commit is checked against what begin was told and not against the
// caller's second claim.
//
// Every write here is gated three times: the family's upload letter (b
// for tickets, wiki pages and technotes; B for forum posts), the bound
// bucket, and the policy — the limit and the type allowlist. What it
// cannot gate is the bytes, which it never sees; that is the held rule
// below, and the steward's scanner on their own bucket.
package api
import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"io"
"log"
"mime"
"net/http"
"path"
"strings"
"time"
"craftdesign.group/quarry/internal/attach"
"craftdesign.group/quarry/internal/index"
"craftdesign.group/quarry/internal/oltp"
"craftdesign.group/quarry/internal/ops"
"craftdesign.group/quarry/internal/s3"
"craftdesign.group/quarry/internal/zitcli"
)
// AttachmentStore is Phase 3's forge-local state: upload tickets, holds
// and the binding's two policy switches. nil is a forge without an
// OLTP, where the write ops answer unavailable and every list is
// unheld.
type AttachmentStore interface {
PutUploadTicket(t oltp.UploadTicket) (string, error)
UploadTicket(id string) (oltp.UploadTicket, bool, error)
DeleteUploadTicket(id string) error
HoldAttachment(h oltp.Hold) error
ReleaseHold(repo, artifact string) error
Holds(repo string) ([]oltp.Hold, error)
HeldArtifacts(repo string) (map[string]bool, error)
SetBucketPolicy(repo string, archives, deleteObjects bool) error
}
const (
// uploadExpiry is how long a presigned PUT is good for. Fifteen
// minutes: long enough for a slow link to move a hundred megabytes,
// short enough that a leaked URL is not a standing door.
uploadExpiry = 15 * time.Minute
// linkExpiry is a presigned GET's life: the hour the RFC names, so a
// private repository's file cannot be shared onward for long.
linkExpiry = time.Hour
// maxFormUploadBytes bounds what the no-script form path holds in
// memory. A presigned PUT signs the content length, so the bytes
// must be counted before they are sent, and a multipart part's
// length is unknown until it has been read; the scripted path knows
// it from the File object and has no such bound.
maxFormUploadBytes = 32 << 20
)
// attachFamily is which kind of object an A card's target names, and
// with it which letters apply (RFC 0008: b for tickets, wiki pages and
// technotes; B for forum posts; q, l and 5 to vouch).
type attachFamily struct {
Family string // ticket | wiki | technote | forum
// Target is the A card's target as the index knows it — the FULL
// ticket id where the caller may have passed a prefix.
Target string
}
func (f attachFamily) uploadLetter() string {
if f.Family == "forum" {
return "B"
}
return "b"
}
func (f attachFamily) vouchLetter() string {
switch f.Family {
case "ticket":
return "q"
case "forum":
return "5"
}
return "l"
}
// attachFamilyOf resolves a target against the index. A wiki page is
// found by its exact name first, since a page may be called anything;
// the three hash-addressed kinds follow. false means no object of any
// kind answers, which is the one refusal that must never be guessed
// around: zit writes an A card for any target string.
func attachFamilyOf(ix *index.Index, target string) (attachFamily, bool, error) {
if target == "" {
return attachFamily{}, false, nil
}
if _, ok, err := ix.WikiPage(target); err != nil || ok {
return attachFamily{Family: "wiki", Target: target}, ok, err
}
if t, ok, err := ix.TicketByPrefixAll(target); err != nil || ok {
return attachFamily{Family: "ticket", Target: t.ID}, ok, err
}
if n, ok, err := ix.TechnoteByEvent(target); err != nil || ok {
return attachFamily{Family: "technote", Target: n.Event}, ok, err
}
if v, ok, err := ix.ForumPostVersions(target); err != nil || ok {
f := attachFamily{Family: "forum", Target: target}
if ok && len(v) > 0 && strings.HasPrefix(v[len(v)-1].Hash, target) {
f.Target = v[len(v)-1].Hash
}
return f, ok, err
}
return attachFamily{}, false, nil
}
// holds answers whether the caller holds any of these letters on repo.
// The edge resolved them into the context when a capability plane
// exists; without one every caller is the operator, which is the rule
// TrustedAuthor and the router already apply.
func (d Deps) holds(ctx context.Context, repo, letters string) bool {
if ops.HasCaps(ctx) {
return ops.CapsOf(ctx).HasAny(letters)
}
if d.CapsFor != nil {
return d.CapsFor(author(ctx), repo).HasAny(letters)
}
return true
}
// attachWriteRepo is repoFor for a write: the repository must exist,
// accept writes, and have a store to write to.
func (d Deps) attachWriteRepo(ctx context.Context, name string) (*Repo, error) {
repo, err := d.repoFor(ctx, name)
if err != nil {
return nil, err
}
if m, _ := d.posture(name); m.Tracking {
return nil, fmt.Errorf("%w: %s mirrors %s and accepts no writes, uploads included", ops.ErrForbidden, name, m.URL)
}
if repo.Zit == nil {
return nil, fmt.Errorf("this repository has no store to write to: %w", ops.ErrUnavailable)
}
return repo, nil
}
// attachTarget resolves the target and checks the family's upload
// letter, the two gates every write here shares.
func (d Deps) attachTarget(ctx context.Context, repo *Repo, name, target string) (attachFamily, error) {
f, ok, err := attachFamilyOf(repo.Index, target)
if err != nil {
return f, err
}
if !ok {
return f, fmt.Errorf("no ticket, wiki page, technote or forum post %q: %w", target, ops.ErrNotFound)
}
if !d.holds(ctx, name, f.uploadLetter()) {
return f, fmt.Errorf("%w: attaching to a %s needs the %s capability", ops.ErrForbidden, f.Family, f.uploadLetter())
}
return f, nil
}
// boundBucket is the binding uploads to name use, refusing when there
// is none. The s3 handle carries the secret and never leaves this
// file's functions.
func (d Deps) boundBucket(name string) (BucketInfo, oltp.BucketBinding, s3.Bucket, error) {
if d.Buckets == nil {
return BucketInfo{}, oltp.BucketBinding{}, s3.Bucket{}, fmt.Errorf("this forge binds no buckets: %w", ops.ErrUnavailable)
}
info, b, err := d.bucketOf(name)
if err != nil {
return info, b, s3.Bucket{}, err
}
if !info.Bound {
return info, b, s3.Bucket{}, fmt.Errorf("%s has no bucket bound for uploads: %w", name, ops.ErrInvalid)
}
return info, b, s3.Bucket{Endpoint: b.Endpoint, Name: b.Bucket, Region: b.Region, AccessKey: b.AccessKey, Secret: b.Secret}, nil
}
// attachClient is what the forge speaks to a bucket with: bounded, so
// a stalled provider is an error and not a held request.
func attachClient() *http.Client { return &http.Client{Timeout: probeTimeout} }
// cleanAttachName is the filename as the A card will carry it: one
// path element, so ../../etc/passwd attaches as passwd.
func cleanAttachName(name string) string {
name = path.Base(strings.ReplaceAll(strings.TrimSpace(name), `\`, "/"))
if name == "." || name == "/" {
return ""
}
return name
}
// withHolds marks which of a target's attachments are held. The index
// knows nothing of holds — they are forge-local — so every list that
// leaves the ops layer passes through here.
func (d Deps) withHolds(repo string, att []index.Attachment) []index.Attachment {
if d.Attachments == nil || len(att) == 0 {
return att
}
held, err := d.Attachments.HeldArtifacts(repo)
if err != nil {
log.Printf("attachments: %s: reading holds: %v", repo, err)
return att
}
for i := range att {
att[i].Held = held[att[i].Hash]
}
return att
}
// attachmentsOn is AttachmentsFor plus the holds: the one call every
// object's op makes for its files.
func (d Deps) attachmentsOn(repo *Repo, name, target string) ([]index.Attachment, error) {
att, err := repo.Index.AttachmentsFor(target)
if err != nil {
return nil, err
}
return d.withHolds(name, att), nil
}
type attachmentBeginIn struct {
Repo string `json:"repo" validate:"required,max=64"`
Target string `json:"target" validate:"required,max=200"`
Name string `json:"name" validate:"required,max=255"`
Size int64 `json:"size" validate:"min=1"`
Type string `json:"type" validate:"required,max=128"`
SHA256 string `json:"sha256" validate:"required,min=64,max=64"`
}
type attachmentBeginOut struct {
Ticket string `json:"ticket"`
// URL is the presigned PUT; Method and Headers are what the request
// must carry exactly, since both headers are in the signature.
URL string `json:"url"`
Method string `json:"method"`
Headers map[string]string `json:"headers"`
Expires string `json:"expires"`
}
type attachmentCommitIn struct {
Repo string `json:"repo" validate:"required,max=64"`
Ticket string `json:"ticket" validate:"required,max=64"`
}
type attachmentCommitOut struct {
Artifact string `json:"artifact"`
Blob string `json:"blob"`
Name string `json:"name"`
Size int64 `json:"size"`
Type string `json:"type"`
SHA256 string `json:"sha256"`
// Held says the upload waits for a person with the family's
// moderation letter: it is in the DAG, and on quarry's pages it is
// a link with a name until then.
Held bool `json:"held"`
}
type attachmentRemoveIn struct {
Repo string `json:"repo" validate:"required,max=64"`
Target string `json:"target" validate:"required,max=200"`
Name string `json:"name" validate:"required,max=255"`
}
type attachmentRemoveOut struct {
Artifact string `json:"artifact"`
Target string `json:"target"`
Name string `json:"name"`
Removed bool `json:"removed"`
// Deleted says the object left the bucket too, which happens only
// for a stub and only when the repository's setting says so.
Deleted bool `json:"deleted"`
}
type attachmentVouchIn struct {
Repo string `json:"repo" validate:"required,max=64"`
Artifact string `json:"artifact" validate:"required,min=4,max=64"`
}
type attachmentVouchOut struct {
Artifact string `json:"artifact"`
Held bool `json:"held"`
}
type attachmentHeldIn struct {
Repo string `json:"repo" validate:"required,max=64"`
}
type attachmentHeldOut struct {
Holds []oltp.Hold `json:"holds"`
}
type attachmentLinkIn struct {
Repo string `json:"repo" validate:"required,max=64"`
// Hash is the CONTENT artifact, the address every attachment route
// takes.
Hash string `json:"hash" validate:"required,min=4,max=64"`
}
// attachmentLinkOut is where a stub's bytes are reachable. Download is
// always answered; Inline only when a person vouched for the file and
// its type is on the inline allowlist, because that URL is the one a
// page puts in an <img> or <video>.
type attachmentLinkOut struct {
External bool `json:"external"`
Name string `json:"name"`
Type string `json:"type,omitempty"`
Size int64 `json:"size,omitempty"`
Held bool `json:"held,omitempty"`
Download string `json:"download,omitempty"`
Inline string `json:"inline,omitempty"`
Expires string `json:"expires,omitempty"`
}
type bucketLimitIn struct {
Repo string `json:"repo" validate:"required,max=64"`
}
// bucketLimitOut is the uploader's view of the binding: whether there
// is one and how big a file may be. Nothing else about the bucket
// reaches anyone but an administrator.
type bucketLimitOut struct {
Repo string `json:"repo"`
Bound bool `json:"bound"`
Limit int64 `json:"limit,omitempty"`
}
type bucketPolicyIn struct {
Repo string `json:"repo" validate:"required,max=64"`
Archives bool `json:"archives"`
DeleteObjects bool `json:"deleteObjects"`
}
// remove is the removal itself, shared by attachment.remove and
// attachment.reject: the sourceless A card, the hold released, and the
// object deleted when the repository's switch says so.
func (d Deps) remove(ctx context.Context, repo *Repo, name string, f attachFamily, filename string) (attachmentRemoveOut, error) {
live, err := repo.Index.AttachmentsFor(f.Target)
if err != nil {
return attachmentRemoveOut{}, err
}
var row *index.Attachment
for i := range live {
if live[i].Name == filename {
row = &live[i]
break
}
}
if row == nil {
return attachmentRemoveOut{}, fmt.Errorf("nothing named %q is attached to %s: %w", filename, f.Target, ops.ErrNotFound)
}
artifact, err := repo.Zit.AttachRm(ctx, f.Target, filename, author(ctx))
if err != nil {
if errors.Is(err, zitcli.ErrNoEntry) {
return attachmentRemoveOut{}, fmt.Errorf("%v: %w", err, ops.ErrInvalid)
}
return attachmentRemoveOut{}, err
}
out := attachmentRemoveOut{Artifact: artifact, Target: f.Target, Name: filename, Removed: true}
if d.Attachments != nil {
if err := d.Attachments.ReleaseHold(name, row.Hash); err != nil {
log.Printf("attachments: %s: releasing hold on %s: %v", name, row.Hash, err)
}
}
if row.External && row.SHA256 != "" {
// Deletion is the steward's decision (RFC 0016 §Writing); the
// switch on the binding is where they made it. Another live
// stub naming the same bytes keeps the object: the key is the
// content, not the attachment.
if _, b, bucket, err := d.boundBucket(name); err == nil && b.DeleteObjects && !d.otherStubNames(repo, row.SHA256) {
if err := bucket.Delete(ctx, attachClient(), attach.Key(row.SHA256), time.Now()); err != nil {
log.Printf("attachments: %s: deleting %s: %v", name, attach.Key(row.SHA256), err)
} else {
out.Deleted = true
}
}
}
return out, nil
}
// otherStubNames reports whether a live stub other than the one being
// removed still names sha. Searched through the live rows of every
// target is not possible cheaply, so this asks the row store the one
// question it can answer: is any live attachment's src a stub with
// this hash. The removal just written is not yet folded, so it still
// counts as one; two or more means another does.
func (d Deps) otherStubNames(repo *Repo, sha string) bool {
n, err := repo.Index.LiveStubCount(sha)
return err == nil && n > 1
}
// registerAttachWrite mounts the Phase 3 ops. Registered whether or not
// a store is wired, like the bucket ops, so the documented surface is
// the same on every forge and an unwired one answers unavailable.
func registerAttachWrite(r *ops.Registry, d Deps) error {
unwired := func() error {
if d.Attachments == nil {
return fmt.Errorf("this forge takes no uploads: %w", ops.ErrUnavailable)
}
return nil
}
if err := ops.Register(r, ops.Op[attachmentBeginIn, attachmentBeginOut]{
Name: "attachment.begin",
// Either upload letter reaches the surface; which one the
// target needs is decided inside, once the target's family is
// known (b for tickets, wiki pages and technotes, B for posts).
Caps: "bB",
Doc: "Begin an upload (RFC 0016 Phase 3): name the target, the file's name, size, type and SHA-256, and receive a presigned PUT — valid fifteen minutes, its content length and type signed — and a ticket for `attachment.commit`. Refused when the target is unknown or read-only, no bucket is bound, the size is over the binding's limit, or the type is not on the policy (text, the common images, PDF, JSON, the common audio and video; archives only when the repository allows them; executables never).",
Kind: ops.Command,
Auth: ops.Token,
Route: "POST /api/v1/r/{repo}/attachments/begin",
Run: func(ctx context.Context, in attachmentBeginIn) (attachmentBeginOut, error) {
if err := unwired(); err != nil {
return attachmentBeginOut{}, err
}
repo, err := d.attachWriteRepo(ctx, in.Repo)
if err != nil {
return attachmentBeginOut{}, err
}
f, err := d.attachTarget(ctx, repo, in.Repo, in.Target)
if err != nil {
return attachmentBeginOut{}, err
}
name := cleanAttachName(in.Name)
if name == "" {
return attachmentBeginOut{}, fmt.Errorf("the attachment needs a filename: %w", ops.ErrInvalid)
}
sha := strings.ToLower(in.SHA256)
if strings.Trim(sha, "0123456789abcdef") != "" {
return attachmentBeginOut{}, fmt.Errorf("sha256 must be 64 hex digits: %w", ops.ErrInvalid)
}
_, b, bucket, err := d.boundBucket(in.Repo)
if err != nil {
return attachmentBeginOut{}, err
}
if in.Size > b.LimitBytes {
return attachmentBeginOut{}, fmt.Errorf("%d bytes is over this repository's limit of %d: %w", in.Size, b.LimitBytes, ops.ErrInvalid)
}
ctype, _, err := mime.ParseMediaType(in.Type)
if err != nil || !attach.Allowed(ctype, b.Archives) {
return attachmentBeginOut{}, fmt.Errorf("files of type %q cannot be attached here: %w", in.Type, ops.ErrInvalid)
}
now := time.Now()
put, err := bucket.Presign(http.MethodPut, attach.Key(sha), uploadExpiry, now, map[string]string{
"content-length": fmt.Sprint(in.Size),
"content-type": ctype,
})
if err != nil {
return attachmentBeginOut{}, err
}
ticket, err := d.Attachments.PutUploadTicket(oltp.UploadTicket{
Repo: in.Repo, Target: f.Target, Name: name, Size: in.Size, Type: ctype, SHA256: sha, Who: author(ctx)})
if err != nil {
return attachmentBeginOut{}, err
}
return attachmentBeginOut{
Ticket: ticket, URL: put.String(), Method: http.MethodPut,
Headers: map[string]string{"Content-Length": fmt.Sprint(in.Size), "Content-Type": ctype},
Expires: now.Add(uploadExpiry).UTC().Format(time.RFC3339),
}, nil
},
}); err != nil {
return err
}
if err := ops.Register(r, ops.Op[attachmentCommitIn, attachmentCommitOut]{
Name: "attachment.commit",
Caps: "bB",
Doc: "Finish an upload: quarry HEADs the object and refuses if it is absent or its length differs from what `attachment.begin` was told, then writes the stub and the A card through zit and answers both hashes. `held` is true when the caller lacks the family's moderation letter (q on tickets, l on the wiki and notes, 5 on the forum) and is not an administrator: the file is in the DAG and shown as a link until a moderator vouches for it.",
Kind: ops.Command,
Auth: ops.Token,
Route: "POST /api/v1/r/{repo}/attachments/commit",
Run: func(ctx context.Context, in attachmentCommitIn) (attachmentCommitOut, error) {
if err := unwired(); err != nil {
return attachmentCommitOut{}, err
}
repo, err := d.attachWriteRepo(ctx, in.Repo)
if err != nil {
return attachmentCommitOut{}, err
}
t, ok, err := d.Attachments.UploadTicket(in.Ticket)
if err != nil {
return attachmentCommitOut{}, err
}
if !ok || t.Repo != in.Repo {
return attachmentCommitOut{}, fmt.Errorf("no such upload ticket; begin again: %w", ops.ErrNotFound)
}
if t.Who != author(ctx) {
return attachmentCommitOut{}, fmt.Errorf("%w: this ticket was begun by someone else", ops.ErrForbidden)
}
f, err := d.attachTarget(ctx, repo, in.Repo, t.Target)
if err != nil {
return attachmentCommitOut{}, err
}
_, _, bucket, err := d.boundBucket(in.Repo)
if err != nil {
return attachmentCommitOut{}, err
}
size, present, err := bucket.Head(ctx, attachClient(), attach.Key(t.SHA256), time.Now())
if err != nil {
return attachmentCommitOut{}, fmt.Errorf("the bucket could not be asked about the object (%v): %w", err, ops.ErrUnavailable)
}
if !present {
return attachmentCommitOut{}, fmt.Errorf("the object is not in the bucket; the upload did not finish: %w", ops.ErrInvalid)
}
if size != t.Size {
return attachmentCommitOut{}, fmt.Errorf("the object is %d bytes and the upload was declared as %d: %w", size, t.Size, ops.ErrInvalid)
}
stub := attach.Stub{SHA256: t.SHA256, Size: t.Size, Type: t.Type, Name: t.Name}
artifact, blob, err := repo.Zit.AttachAdd(ctx, f.Target, t.Name, t.Type, author(ctx), "", stub.Format())
if err != nil {
if errors.Is(err, zitcli.ErrNoEntry) {
return attachmentCommitOut{}, fmt.Errorf("%v: %w", err, ops.ErrInvalid)
}
return attachmentCommitOut{}, err
}
out := attachmentCommitOut{Artifact: artifact, Blob: blob, Name: t.Name, Size: t.Size, Type: t.Type, SHA256: t.SHA256}
// The held rule (RFC 0016 §Trust): trusted for the family
// means the moderation letter or the steward's own. A hold
// that cannot be recorded is logged and the file shows —
// the A card is already written and cannot be unwritten,
// and a hidden hold row would be a hold nobody can release.
if !d.holds(ctx, in.Repo, f.vouchLetter()+"a") {
if err := d.Attachments.HoldAttachment(oltp.Hold{Repo: in.Repo, Artifact: artifact, Target: f.Target, Name: t.Name, Who: author(ctx)}); err != nil {
log.Printf("attachments: %s: recording hold on %s: %v", in.Repo, artifact, err)
} else {
out.Held = true
}
}
if err := d.Attachments.DeleteUploadTicket(in.Ticket); err != nil {
log.Printf("attachments: %s: spending ticket %s: %v", in.Repo, in.Ticket, err)
}
return out, nil
},
}); err != nil {
return err
}
if err := ops.Register(r, ops.Op[attachmentRemoveIn, attachmentRemoveOut]{
Name: "attachment.remove",
Caps: "bB",
Doc: "Remove an attachment: an A card with no source, fossil's own deletion form. Nothing leaves the DAG. When the repository's setting `deleteObjects` is on and the attachment was an upload, the object is deleted from the bucket too, unless another live attachment names the same bytes.",
Kind: ops.Command,
Auth: ops.Token,
Route: "POST /api/v1/r/{repo}/attachments/remove",
Run: func(ctx context.Context, in attachmentRemoveIn) (attachmentRemoveOut, error) {
repo, err := d.attachWriteRepo(ctx, in.Repo)
if err != nil {
return attachmentRemoveOut{}, err
}
f, err := d.attachTarget(ctx, repo, in.Repo, in.Target)
if err != nil {
return attachmentRemoveOut{}, err
}
return d.remove(ctx, repo, in.Repo, f, in.Name)
},
}); err != nil {
return err
}
resolveHeld := func(ctx context.Context, name, artifact string) (*Repo, index.Attachment, attachFamily, error) {
if err := unwired(); err != nil {
return nil, index.Attachment{}, attachFamily{}, err
}
repo, err := d.attachWriteRepo(ctx, name)
if err != nil {
return nil, index.Attachment{}, attachFamily{}, err
}
row, ok, err := repo.Index.AttachmentByHash(artifact)
if err != nil {
return nil, index.Attachment{}, attachFamily{}, err
}
if !ok {
return nil, index.Attachment{}, attachFamily{}, fmt.Errorf("no attachment artifact %q: %w", artifact, ops.ErrNotFound)
}
f, ok, err := attachFamilyOf(repo.Index, row.Target)
if err != nil {
return nil, index.Attachment{}, attachFamily{}, err
}
if !ok {
return nil, index.Attachment{}, attachFamily{}, fmt.Errorf("the attachment's target %q is unknown: %w", row.Target, ops.ErrNotFound)
}
if !d.holds(ctx, name, f.vouchLetter()+"a") {
return nil, index.Attachment{}, attachFamily{}, fmt.Errorf("%w: reviewing a %s attachment needs the %s capability", ops.ErrForbidden, f.Family, f.vouchLetter())
}
return repo, row, f, nil
}
if err := ops.Register(r, ops.Op[attachmentVouchIn, attachmentVouchOut]{
Name: "attachment.vouch",
Caps: "ql5a",
Doc: "Vouch for a held upload: it renders from now on. Needs the family's moderation letter (q, l or 5) or the administrator's.",
Kind: ops.Command,
Auth: ops.Token,
Route: "POST /api/v1/r/{repo}/attachments/vouch",
Run: func(ctx context.Context, in attachmentVouchIn) (attachmentVouchOut, error) {
_, row, _, err := resolveHeld(ctx, in.Repo, in.Artifact)
if err != nil {
return attachmentVouchOut{}, err
}
if err := d.Attachments.ReleaseHold(in.Repo, row.Hash); err != nil {
return attachmentVouchOut{}, err
}
return attachmentVouchOut{Artifact: row.Hash}, nil
},
}); err != nil {
return err
}
if err := ops.Register(r, ops.Op[attachmentVouchIn, attachmentRemoveOut]{
Name: "attachment.reject",
Caps: "ql5a",
Doc: "Reject a held upload: the hold is cleared and the attachment removed as `attachment.remove` would, object deletion included when the repository says so.",
Kind: ops.Command,
Auth: ops.Token,
Route: "POST /api/v1/r/{repo}/attachments/reject",
Run: func(ctx context.Context, in attachmentVouchIn) (attachmentRemoveOut, error) {
repo, row, f, err := resolveHeld(ctx, in.Repo, in.Artifact)
if err != nil {
return attachmentRemoveOut{}, err
}
if err := d.Attachments.ReleaseHold(in.Repo, row.Hash); err != nil {
return attachmentRemoveOut{}, err
}
return d.remove(ctx, repo, in.Repo, f, row.Name)
},
}); err != nil {
return err
}
if err := ops.Register(r, ops.Op[attachmentHeldIn, attachmentHeldOut]{
Name: "attachment.held",
Caps: "ql5a",
Doc: "The review queue: uploads on this repository nobody trusted has vouched for, oldest first.",
Kind: ops.Query,
Auth: ops.Token,
Route: "GET /api/v1/r/{repo}/attachments/held",
Run: func(ctx context.Context, in attachmentHeldIn) (attachmentHeldOut, error) {
if err := unwired(); err != nil {
return attachmentHeldOut{}, err
}
if _, err := d.repoFor(ctx, in.Repo); err != nil {
return attachmentHeldOut{}, err
}
holds, err := d.Attachments.Holds(in.Repo)
if err != nil {
return attachmentHeldOut{}, err
}
if holds == nil {
holds = []oltp.Hold{}
}
return attachmentHeldOut{Holds: holds}, nil
},
}); err != nil {
return err
}
if err := ops.Register(r, ops.Op[attachmentLinkIn, attachmentLinkOut]{
Name: "attachment.link",
Caps: "jr",
Doc: "Where an uploaded attachment's bytes are: a link on the bucket's public base when the repository is public and one is set, otherwise a presigned GET good for an hour that names the file and its type. `inline` is answered only for a vouched file whose type may render in a page; SVG never. A real (non-upload) attachment answers `external: false` and is served by quarry's own routes.",
Kind: ops.Query,
Auth: ops.Anonymous,
Route: "GET /api/v1/r/{repo}/attachment/{hash}/link",
Run: func(ctx context.Context, in attachmentLinkIn) (attachmentLinkOut, error) {
repo, err := d.repoFor(ctx, in.Repo)
if err != nil {
return attachmentLinkOut{}, err
}
row, ok, err := repo.Index.AttachmentBySrc(in.Hash)
if err != nil {
return attachmentLinkOut{}, err
}
if !ok {
return attachmentLinkOut{}, fmt.Errorf("no attachment %q: %w", in.Hash, ops.ErrNotFound)
}
out := attachmentLinkOut{External: row.External, Name: row.Name}
if !row.External {
return out, nil
}
out.Type, out.Size = row.Mimetype, row.Size
out.Held = d.withHolds(in.Repo, []index.Attachment{row})[0].Held
info, _, bucket, err := d.boundBucket(in.Repo)
if err != nil {
return attachmentLinkOut{}, fmt.Errorf("%s has no bucket bound, so this file cannot be reached: %w", in.Repo, ops.ErrNotFound)
}
inline := !out.Held && attach.Inline(row.Mimetype)
private := d.Private != nil && d.Private()[in.Repo]
if info.PublicBase != "" && !private {
out.Download = info.PublicBase + "/" + attach.Key(row.SHA256)
if inline {
out.Inline = out.Download
}
return out, nil
}
now := time.Now()
dl, err := bucket.PresignGet(attach.Key(row.SHA256), linkExpiry, now, row.Name, row.Mimetype, false)
if err != nil {
return attachmentLinkOut{}, err
}
out.Download = dl.String()
out.Expires = now.Add(linkExpiry).UTC().Format(time.RFC3339)
if inline {
il, err := bucket.PresignGet(attach.Key(row.SHA256), linkExpiry, now, row.Name, row.Mimetype, true)
if err != nil {
return attachmentLinkOut{}, err
}
out.Inline = il.String()
}
return out, nil
},
}); err != nil {
return err
}
if err := ops.Register(r, ops.Op[bucketLimitIn, bucketLimitOut]{
Name: "bucket.limit",
Caps: "bB",
Doc: "Whether this repository takes uploads and how large one may be — the uploader's question, answered to holders of the upload letters without the binding's detail.",
Kind: ops.Query,
Auth: ops.Token,
Route: "GET /api/v1/r/{repo}/bucket/limit",
Run: func(ctx context.Context, in bucketLimitIn) (bucketLimitOut, error) {
if _, err := d.repoFor(ctx, in.Repo); err != nil {
return bucketLimitOut{}, err
}
if d.Buckets == nil {
return bucketLimitOut{Repo: in.Repo}, nil
}
info, b, err := d.bucketOf(in.Repo)
if err != nil {
return bucketLimitOut{}, err
}
out := bucketLimitOut{Repo: in.Repo, Bound: info.Bound}
if info.Bound {
out.Limit = b.LimitBytes
}
return out, nil
},
}); err != nil {
return err
}
return ops.Register(r, ops.Op[bucketPolicyIn, BucketInfo]{
Name: "bucket.policy",
Caps: "a",
Doc: "The binding's two switches: `archives` opens the upload type policy to zip, gzip and tar; `deleteObjects` makes a removal delete the object from the bucket as well. Both are off by default.",
Kind: ops.Command,
Auth: ops.Admin,
Route: "POST /api/v1/r/{repo}/bucket/policy",
Run: func(ctx context.Context, in bucketPolicyIn) (BucketInfo, error) {
if err := unwired(); err != nil {
return BucketInfo{}, err
}
if _, err := d.repoFor(ctx, in.Repo); err != nil {
return BucketInfo{}, err
}
if parent := d.companionParent(in.Repo); parent != "" {
return BucketInfo{}, fmt.Errorf("%s uses %s's bucket; set its policy there: %w", in.Repo, parent, ops.ErrInvalid)
}
if err := d.Attachments.SetBucketPolicy(in.Repo, in.Archives, in.DeleteObjects); err != nil {
return BucketInfo{}, fmt.Errorf("%v: %w", err, ops.ErrInvalid)
}
what := "attachments policy: archives " + onOff(in.Archives) + ", delete objects on removal " + onOff(in.DeleteObjects)
d.bucketSettingsEvent(ctx, in.Repo, what)
info, _, err := d.bucketOf(in.Repo)
return info, err
},
})
}
// uploadStatus is the router's own mapping of an op error to a status,
// for the one handler here that is not an op.
func uploadStatus(err error) int {
switch {
case errors.Is(err, ops.ErrNotFound):
return http.StatusNotFound
case errors.Is(err, ops.ErrInvalid):
return http.StatusBadRequest
case errors.Is(err, ops.ErrForbidden):
return http.StatusForbidden
case errors.Is(err, ops.ErrUnavailable):
return http.StatusServiceUnavailable
}
return http.StatusInternalServerError
}
func onOff(b bool) string {
if b {
return "on"
}
return "off"
}
// UploadForm is the no-script path (RFC 0016 §Writing): a multipart
// form with `file` (and an optional `name`) that quarry carries to the
// bucket on the reader's behalf — begin, the PUT, commit — through the
// same ops the browser would call, so every gate is the same gate.
//
// The file is held in memory while it is counted and hashed, never on
// disk, and bounded twice: the binding's limit, refused as soon as one
// byte past it arrives, and maxFormUploadBytes, past which the scripted
// path is the way. ctx must carry the caller the way a gated request's
// does; the ops refuse on the letters as they would for any client.
func UploadForm(ctx context.Context, d Deps, r *ops.Registry, repo, target string, req *http.Request) (attachmentCommitOut, error) {
info, b, err := d.bucketOf(repo)
if d.Buckets == nil || err != nil || !info.Bound {
return attachmentCommitOut{}, fmt.Errorf("%s has no bucket bound for uploads: %w", repo, ops.ErrInvalid)
}
limit := min(b.LimitBytes, maxFormUploadBytes)
mr, err := req.MultipartReader()
if err != nil {
return attachmentCommitOut{}, fmt.Errorf("expected a multipart form with a file: %w", ops.ErrInvalid)
}
var body bytes.Buffer
var name, ctype string
sum := sha256.New()
got := false
for {
part, err := mr.NextPart()
if errors.Is(err, io.EOF) {
break
}
if err != nil {
return attachmentCommitOut{}, fmt.Errorf("%v: %w", err, ops.ErrInvalid)
}
switch part.FormName() {
case "name":
text, _ := io.ReadAll(io.LimitReader(part, 512))
if n := cleanAttachName(string(text)); n != "" {
name = n
}
case "file":
if name == "" {
name = cleanAttachName(part.FileName())
}
ctype = part.Header.Get("Content-Type")
n, err := io.Copy(io.MultiWriter(&body, sum), io.LimitReader(part, limit+1))
if err != nil {
return attachmentCommitOut{}, err
}
if n > limit {
if limit < b.LimitBytes {
return attachmentCommitOut{}, fmt.Errorf("files over %d bytes need a browser with script, which sends them to the bucket directly: %w", limit, ops.ErrInvalid)
}
return attachmentCommitOut{}, fmt.Errorf("the file is over this repository's limit of %d bytes: %w", b.LimitBytes, ops.ErrInvalid)
}
got = true
}
part.Close()
}
if !got || name == "" {
return attachmentCommitOut{}, fmt.Errorf("choose a file: %w", ops.ErrInvalid)
}
if body.Len() == 0 {
return attachmentCommitOut{}, fmt.Errorf("the file is empty: %w", ops.ErrInvalid)
}
if ctype == "" || ctype == "application/octet-stream" {
// Browsers name a type from the extension and say octet-stream
// when they have none; the policy then refuses it, which is the
// right answer for a file nobody could name.
if guess := mime.TypeByExtension(strings.ToLower(path.Ext(name))); guess != "" {
ctype = guess
}
}
begin := func() (attachmentBeginOut, error) {
out, err := r.Invoke(ctx, "attachment.begin", attachmentBeginIn{
Repo: repo, Target: target, Name: name, Size: int64(body.Len()), Type: ctype,
SHA256: hex.EncodeToString(sum.Sum(nil))})
if err != nil {
return attachmentBeginOut{}, err
}
return out.(attachmentBeginOut), nil
}
send := func() (attachmentCommitOut, error) {
bo, err := begin()
if err != nil {
return attachmentCommitOut{}, err
}
put, err := http.NewRequestWithContext(ctx, bo.Method, bo.URL, bytes.NewReader(body.Bytes()))
if err != nil {
return attachmentCommitOut{}, err
}
put.ContentLength = int64(body.Len())
put.Header.Set("Content-Type", bo.Headers["Content-Type"])
resp, err := attachClient().Do(put)
if err != nil {
return attachmentCommitOut{}, fmt.Errorf("the bucket refused the upload: %w", ops.ErrUnavailable)
}
io.Copy(io.Discard, io.LimitReader(resp.Body, 1<<16))
resp.Body.Close()
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
return attachmentCommitOut{}, fmt.Errorf("the bucket answered %d to the upload: %w", resp.StatusCode, ops.ErrUnavailable)
}
out, err := r.Invoke(ctx, "attachment.commit", attachmentCommitIn{Repo: repo, Ticket: bo.Ticket})
if err != nil {
return attachmentCommitOut{}, err
}
return out.(attachmentCommitOut), nil
}
if d.Bulk == nil {
return send()
}
var out attachmentCommitOut
err = d.Bulk.Do(ctx, func() error {
var err error
out, err = send()
return err
})
return out, err
}
// UploadReceiver is UploadForm on the API's own path, for a client
// with Basic credentials and no script: the same gates, authenticated
// the way every op under /api/ is.
func UploadReceiver(d Deps, r *ops.Registry, verify Verify, capsFor CapsFor) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
repo, target := req.PathValue("repo"), req.PathValue("target")
ctx := req.Context()
if verify != nil {
user, pass, _ := req.BasicAuth()
switch verify(repo, user, pass, ops.Token) {
case Denied:
w.Header().Set("WWW-Authenticate", `Basic realm="quarry"`)
writeError(w, http.StatusUnauthorized, errors.New("authentication required"))
return
case Insufficient:
writeError(w, http.StatusForbidden, errors.New("this account does not have the capability this request needs"))
return
}
ctx = ops.WithUser(ctx, user)
if capsFor != nil {
ctx = ops.WithCaps(ctx, capsFor(user, repo))
}
}
out, err := UploadForm(ctx, d, r, repo, target, req)
if err != nil {
writeError(w, uploadStatus(err), err)
return
}
writeJSON(w, http.StatusOK, out)
})
}