// attachwrite.go is RFC 0016 Phase 3: writing an attachment through the
// forge. The browser never sends the file to quarry — begin mints a
// presigned PUT whose signature fixes the key, the length and the type;
// the browser PUTs; commit checks what landed and writes the stub
// through zit. What the forge holds between the two is a ticket, so
// commit is checked against what begin was told and not against the
// caller's second claim.
//
// Every write here is gated three times: the family's upload letter (b
// for tickets, wiki pages and technotes; B for forum posts), the bound
// bucket, and the policy — the limit and the type allowlist. What it
// cannot gate is the bytes, which it never sees; that is the held rule
// below, and the steward's scanner on their own bucket.
package api
import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"io"
"log"
"mime"
"net/http"
"path"
"strings"
"time"
"craftdesign.group/quarry/internal/attach"
"craftdesign.group/quarry/internal/index"
"craftdesign.group/quarry/internal/oltp"
"craftdesign.group/quarry/internal/ops"
"craftdesign.group/quarry/internal/s3"
"craftdesign.group/quarry/internal/zitcli"
)
// AttachmentStore is Phase 3's forge-local state: upload tickets, holds
// and the binding's two policy switches. nil is a forge without an
// OLTP, where the write ops answer unavailable and every list is
// unheld.
type AttachmentStore interface {
PutUploadTicket(t oltp.UploadTicket) (string, error)
UploadTicket(id string) (oltp.UploadTicket, bool, error)
DeleteUploadTicket(id string) error
HoldAttachment(h oltp.Hold) error
ReleaseHold(repo, artifact string) error
Holds(repo string) ([]oltp.Hold, error)
HeldArtifacts(repo string) (map[string]bool, error)
SetBucketPolicy(repo string, archives, deleteObjects bool) error
}
const (
// uploadExpiry is how long a presigned PUT is good for. Fifteen
// minutes: long enough for a slow link to move a hundred megabytes,
// short enough that a leaked URL is not a standing door.
uploadExpiry = 15 * time.Minute
// linkExpiry is a presigned GET's life: the hour the RFC names, so a
// private repository's file cannot be shared onward for long.
linkExpiry = time.Hour
// maxFormUploadBytes bounds what the no-script form path holds in
// memory. A presigned PUT signs the content length, so the bytes
// must be counted before they are sent, and a multipart part's
// length is unknown until it has been read; the scripted path knows
// it from the File object and has no such bound.
maxFormUploadBytes = 32 << 20
)
// attachFamily is which kind of object an A card's target names, and
// with it which letters apply (RFC 0008: b for tickets, wiki pages and
// technotes; B for forum posts; q, l and 5 to vouch).
type attachFamily struct {
Family string // ticket | wiki | technote | forum
// Target is the A card's target as the index knows it — the FULL
// ticket id where the caller may have passed a prefix.
Target string
// Local says the target is a ticket on the forge tier: it has no A
// card to hang a file on yet, so an upload against it is HELD in
// the forge database and written to the log when the ticket is
// promoted (9e628be2). The letters are the ticket family's; only
// where the stub goes differs.
Local bool
}
func (f attachFamily) uploadLetter() string {
if f.Family == "forum" {
return "B"
}
return "b"
}
func (f attachFamily) vouchLetter() string {
switch f.Family {
case "ticket":
return "q"
case "forum":
return "5"
}
return "l"
}
// attachFamilyOf resolves a target against the index. A wiki page is
// found by its exact name first, since a page may be called anything;
// the three hash-addressed kinds follow. false means no object of any
// kind answers, which is the one refusal that must never be guessed
// around: zit writes an A card for any target string.
func attachFamilyOf(ix *index.Index, target string) (attachFamily, bool, error) {
if target == "" {
return attachFamily{}, false, nil
}
if _, ok, err := ix.WikiPage(target); err != nil || ok {
return attachFamily{Family: "wiki", Target: target}, ok, err
}
if t, ok, err := ix.TicketByPrefixAll(target); err != nil || ok {
return attachFamily{Family: "ticket", Target: t.ID}, ok, err
}
if n, ok, err := ix.TechnoteByEvent(target); err != nil || ok {
return attachFamily{Family: "technote", Target: n.Event}, ok, err
}
if v, ok, err := ix.ForumPostVersions(target); err != nil || ok {
f := attachFamily{Family: "forum", Target: target}
if ok && len(v) > 0 && strings.HasPrefix(v[len(v)-1].Hash, target) {
f.Target = v[len(v)-1].Hash
}
return f, ok, err
}
return attachFamily{}, false, nil
}
// attachFamilyFor is attachFamilyOf plus the forge tier: a target the
// index does not hold may still be a local ticket, which takes
// attachments like any other ticket and holds them until promotion.
// The index is asked first, so a promoted ticket resolves to its
// artifact and never to the tombstone it left behind.
func (d Deps) attachFamilyFor(repo *Repo, name, target string) (attachFamily, bool, error) {
f, ok, err := attachFamilyOf(repo.Index, target)
if err != nil || ok || d.Local == nil {
return f, ok, err
}
lt, ok, err := d.Local.LocalTicketByPrefix(name, strings.TrimPrefix(target, "tkt:"))
if err != nil || !ok {
return attachFamily{}, false, err
}
return attachFamily{Family: "ticket", Target: lt.ID, Local: true}, true, nil
}
// localAttachmentRows presents what a local ticket holds in the shape
// every attachment reader already takes. A held file has no artifact,
// so Hash is empty and Src is the object's key: the address
// attachment.head and attachment.link answer for it, and the one the
// pages link through.
func (d Deps) localAttachmentRows(name, ticket string) ([]index.Attachment, error) {
if d.Local == nil {
return nil, nil
}
held, err := d.Local.LocalAttachments(name, ticket)
if err != nil {
return nil, err
}
out := make([]index.Attachment, 0, len(held))
for _, a := range held {
out = append(out, index.Attachment{
Name: a.Name, Target: a.Ticket, Src: a.SHA256, Date: a.Created, User: a.Who,
Mimetype: a.Type, External: true, SHA256: a.SHA256, Size: a.Size,
})
}
return out, nil
}
// localAttachmentBySrc finds a held file by its object key, for the
// read ops that take an attachment's address: they look in the index
// first and here second, so a held file reads like any upload.
func (d Deps) localAttachmentBySrc(name, src string) (index.Attachment, bool) {
if d.Local == nil {
return index.Attachment{}, false
}
a, ok, err := d.Local.LocalAttachmentBySHA(name, src)
if err != nil || !ok {
return index.Attachment{}, false
}
return index.Attachment{
Name: a.Name, Target: a.Ticket, Src: a.SHA256, Date: a.Created, User: a.Who,
Mimetype: a.Type, External: true, SHA256: a.SHA256, Size: a.Size,
}, true
}
// holds answers whether the caller holds any of these letters on repo.
// The edge resolved them into the context when a capability plane
// exists; without one every caller is the operator, which is the rule
// TrustedAuthor and the router already apply.
func (d Deps) holds(ctx context.Context, repo, letters string) bool {
if ops.HasCaps(ctx) {
return ops.CapsOf(ctx).HasAny(letters)
}
if d.CapsFor != nil {
return d.CapsFor(author(ctx), repo).HasAny(letters)
}
return true
}
// attachWriteRepo is repoFor for a write: the repository must exist,
// accept writes, and have a store to write to.
func (d Deps) attachWriteRepo(ctx context.Context, name string) (*Repo, error) {
repo, err := d.repoFor(ctx, name)
if err != nil {
return nil, err
}
if m, _ := d.posture(name); m.Tracking {
return nil, fmt.Errorf("%w: %s mirrors %s and accepts no writes, uploads included", ops.ErrForbidden, name, m.URL)
}
if repo.Zit == nil {
return nil, fmt.Errorf("this repository has no store to write to: %w", ops.ErrUnavailable)
}
return repo, nil
}
// noSuchTarget explains a target neither the store nor the forge tier
// holds.
//
// A forge-tier ticket now takes attachments (attachFamilyFor), so the
// one local case left is a ticket that has BEEN promoted: its local id
// is a redirect, and the files belong on the artifact it became. The
// bare not-found read like the ticket did not exist, which is the
// refusal that must not lie.
func (d Deps) noSuchTarget(repo, target string) error {
if d.Local != nil {
if to, ok := d.Local.LocalTicketRedirect(repo, strings.TrimPrefix(target, "tkt:")); ok {
return fmt.Errorf(
"%w: ticket %.12s was promoted to %.12s; attach to that", ops.ErrNotFound, target, to)
}
}
return fmt.Errorf("no ticket, wiki page, technote or forum post %q: %w", target, ops.ErrNotFound)
}
// attachTarget resolves the target and checks the family's upload
// letter, the two gates every write here shares.
func (d Deps) attachTarget(ctx context.Context, repo *Repo, name, target string) (attachFamily, error) {
f, ok, err := d.attachFamilyFor(repo, name, target)
if err != nil {
return f, err
}
if !ok {
return f, d.noSuchTarget(name, target)
}
if !d.holds(ctx, name, f.uploadLetter()) {
return f, fmt.Errorf("%w: attaching to a %s needs the %s capability", ops.ErrForbidden, f.Family, f.uploadLetter())
}
return f, nil
}
// boundBucket is the binding uploads to name use, refusing when there
// is none. The s3 handle carries the secret and never leaves this
// file's functions.
func (d Deps) boundBucket(name string) (BucketInfo, oltp.BucketBinding, s3.Bucket, error) {
if d.Buckets == nil {
return BucketInfo{}, oltp.BucketBinding{}, s3.Bucket{}, fmt.Errorf("this forge binds no buckets: %w", ops.ErrUnavailable)
}
info, b, err := d.bucketOf(name)
if err != nil {
return info, b, s3.Bucket{}, err
}
if !info.Bound {
// The forge fallback arrives here already unbound, with its
// reason in words; anything else is plainly unbound.
if info.Note != "" {
return info, b, s3.Bucket{}, fmt.Errorf("%s: %w", info.Note, ops.ErrInvalid)
}
return info, b, s3.Bucket{}, fmt.Errorf("%s has no bucket bound for uploads: %w", name, ops.ErrInvalid)
}
// A repository's files do not go to the FORGE's bucket.
//
// forgeAnswer now reports the fallback as unbound, so the refusal
// above is the one a caller meets; this check stays as the
// guarantee that the bytes never go there even if that changes.
//
// bucketOf falls back to the forge's binding when a repository has
// none of its own, so uploads silently landed in the operator's
// storage: dmestas attached files to an announcement on quarry and
// found them in his personal R2 (9b8b8fb6). The disclosure half of
// that — a repo admin being handed the endpoint, bucket name and
// key id — is already fixed by forgeAnswer. This is the other
// half, which is where the BYTES went.
//
// dmestas's call was to refuse and say what to do. It is worth
// recording what that costs, because the fallback was not
// thoughtless: a forge with no per-repository storage now cannot
// attach anything until somebody binds a bucket, and the comment
// in bucketOf argues that makes a self-hosted forge unusable by
// anybody but its operator. The answer is that silently filing a
// stranger's uploads into the operator's account is worse than a
// refusal that names the remedy — an unusable feature is visible,
// and a misrouted one is not.
//
// Marks are untouched. An avatar is forge-level furniture rather
// than a repository's content, and forgeBucketForAvatar resolves
// the forge binding on purpose, so a fresh forge still draws
// pictures.
if info.Forge {
return info, b, s3.Bucket{}, fmt.Errorf("%s: %w", NoBucketOfItsOwn(name), ops.ErrInvalid)
}
return info, b, s3.Bucket{Endpoint: b.Endpoint, Name: b.Bucket, Region: b.Region, AccessKey: b.AccessKey, Secret: b.Secret}, nil
}
// NoBucketOfItsOwn is why an upload cannot be accepted at a repository
// whose binding resolved to the FORGE's.
//
// One sentence, named once, because two surfaces answer this question and
// they disagreed: bucket.limit reported the resolved binding as `bound`
// and handed out a size limit, while begin refused through boundBucket —
// so a reader was told they were ready and then failed (G-130). The read
// side now reports this same sentence instead of promising an upload that
// will not happen.
//
// bucketOf falls back to the forge's binding so a repository can inherit
// storage; the refusal is deliberate and its reasoning is in boundBucket,
// where the bytes-to-the-operator's-account incident is recorded.
func NoBucketOfItsOwn(name string) string {
return name + " has no bucket of its own, and a repository's uploads do not go to the " +
"forge's storage: bind a bucket under Settings > Attachments"
}
// attachClient is what the forge speaks to a bucket with: bounded, so
// a stalled provider is an error and not a held request.
func attachClient() *http.Client { return &http.Client{Timeout: probeTimeout} }
// cleanAttachName is the filename as the A card will carry it: one
// path element, so ../../etc/passwd attaches as passwd.
func cleanAttachName(name string) string {
name = path.Base(strings.ReplaceAll(strings.TrimSpace(name), `\`, "/"))
if name == "." || name == "/" {
return ""
}
return name
}
// logModeration records an attachment decision in the same public log
// the forum's decisions go to.
//
// Until now nothing recorded them. attachment.reject clears the hold
// and removes the file, and the removal does write a sourceless A card
// carrying the actor — so the DAG knew WHO and WHEN — but there was no
// reason anywhere, and no surface: a person would have had to go
// looking in the DAG for a control artifact. Meanwhile the forum, for
// the same decision, appends a line with actor, action, target and
// reason that reads anonymously. Same product, same word, one record
// between the two (01a3b35f4055).
//
// Best effort. A decision that happened must not be undone because its
// log line could not be written, and the forum's own call sites take
// the same view — the alternative is refusing to moderate because the
// audit trail is unavailable.
func (d Deps) logModeration(repo, actor, action, name, artifact, reason, about string) {
if d.Forum == nil {
return
}
target := "attachment:" + name
if len(artifact) >= 12 {
target += "@" + artifact[:12]
}
if err := d.Forum.AppendModLog(repo, actor, action, target, reason, about); err != nil {
log.Printf("attachments: %s: moderation log: %v", repo, err)
}
}
// withHolds marks which of a target's attachments are held. The index
// knows nothing of holds — they are forge-local — so every list that
// leaves the ops layer passes through here.
func (d Deps) withHolds(repo string, att []index.Attachment) []index.Attachment {
if d.Attachments == nil || len(att) == 0 {
return att
}
held, err := d.Attachments.HeldArtifacts(repo)
if err != nil {
log.Printf("attachments: %s: reading holds: %v", repo, err)
return att
}
for i := range att {
att[i].Held = held[att[i].Hash]
}
return att
}
// attachmentsOn is AttachmentsFor plus the holds: the one call every
// object's op makes for its files.
func (d Deps) attachmentsOn(repo *Repo, name, target string) ([]index.Attachment, error) {
att, err := repo.Index.AttachmentsFor(target)
if err != nil {
return nil, err
}
return d.withHolds(name, att), nil
}
type attachmentBeginIn struct {
Repo string `json:"repo" validate:"required,max=64"`
// Target names the object the file attaches to. Empty DEFERS the
// ticket: the object does not exist yet (a thread, a reply or a
// ticket being written), Family says which kind it will be, and
// the op that writes the object — or `attachment.commit` — names
// the target. Same fifteen-minute PUT, same hour on the ticket.
Target string `json:"target" validate:"max=200"`
Family string `json:"family" validate:"max=16"`
Name string `json:"name" validate:"required,max=255"`
Size int64 `json:"size" validate:"min=1"`
Type string `json:"type" validate:"required,max=128"`
SHA256 string `json:"sha256" validate:"required,min=64,max=64"`
}
type attachmentBeginOut struct {
Ticket string `json:"ticket"`
// URL is the presigned PUT; Method and Headers are what the request
// must carry exactly, since both headers are in the signature.
URL string `json:"url"`
Method string `json:"method"`
Headers map[string]string `json:"headers"`
Expires string `json:"expires"`
}
type attachmentCommitIn struct {
Repo string `json:"repo" validate:"required,max=64"`
Ticket string `json:"ticket" validate:"required,max=64"`
// Target is required for a deferred ticket and ignored otherwise:
// a ticket begun against an object commits to that object.
Target string `json:"target" validate:"max=200"`
}
type attachmentCommitOut struct {
Artifact string `json:"artifact"`
Blob string `json:"blob"`
Name string `json:"name"`
Size int64 `json:"size"`
Type string `json:"type"`
SHA256 string `json:"sha256"`
// Held says the upload waits for a person with the family's
// moderation letter: it is in the DAG, and on quarry's pages it is
// a link with a name until then.
Held bool `json:"held"`
// Local says the target is a forge-tier ticket: the file is in the
// bucket and held against the ticket, with no artifact or blob yet.
// Both are written when the ticket is promoted.
Local bool `json:"local,omitempty"`
}
type attachmentRemoveIn struct {
Repo string `json:"repo" validate:"required,max=64"`
Target string `json:"target" validate:"required,max=200"`
Name string `json:"name" validate:"required,max=255"`
}
type attachmentRemoveOut struct {
Artifact string `json:"artifact"`
Target string `json:"target"`
Name string `json:"name"`
Removed bool `json:"removed"`
// Deleted says the object left the bucket too, which happens only
// for a stub and only when the repository's setting says so.
Deleted bool `json:"deleted"`
}
type attachmentVouchIn struct {
Repo string `json:"repo" validate:"required,max=64"`
Artifact string `json:"artifact" validate:"required,min=4,max=64"`
// Reason is the moderator's, recorded in the same public log the
// forum's decisions go to. Optional, and the same bound the forum
// uses, so the two surfaces cannot drift apart on what a reason is.
Reason string `json:"reason" validate:"max=500"`
}
type attachmentVouchOut struct {
Artifact string `json:"artifact"`
Held bool `json:"held"`
}
type attachmentHeldIn struct {
Repo string `json:"repo" validate:"required,max=64"`
}
type attachmentHeldOut struct {
Holds []oltp.Hold `json:"holds"`
}
type attachmentLinkIn struct {
Repo string `json:"repo" validate:"required,max=64"`
// Hash is the CONTENT artifact, the address every attachment route
// takes.
Hash string `json:"hash" validate:"required,min=4,max=64"`
}
// attachmentLinkOut is where a stub's bytes are reachable. Download is
// always answered; Inline only when a person vouched for the file and
// its type is on the inline allowlist, because that URL is the one a
// page puts in an <img> or <video>.
type attachmentLinkOut struct {
External bool `json:"external"`
Name string `json:"name"`
Type string `json:"type,omitempty"`
Size int64 `json:"size,omitempty"`
Held bool `json:"held,omitempty"`
Download string `json:"download,omitempty"`
Inline string `json:"inline,omitempty"`
Expires string `json:"expires,omitempty"`
}
type bucketLimitIn struct {
Repo string `json:"repo" validate:"required,max=64"`
}
// bucketLimitOut is the uploader's view of the binding: whether there
// is one and how big a file may be. Nothing else about the bucket
// reaches anyone but an administrator.
type bucketLimitOut struct {
Repo string `json:"repo"`
Bound bool `json:"bound"`
Limit int64 `json:"limit,omitempty"`
// Note says why an upload will NOT work when Bound is false and the
// reason is knowable — the same sentence `begin` refuses with, so a
// reader learns the remedy from the read instead of from a failure.
Note string `json:"note,omitempty"`
}
type bucketPolicyIn struct {
Repo string `json:"repo" validate:"required,max=64"`
// POINTERS, so absent is not the same as false.
//
// These were plain bools, and SetBucketPolicy was handed them
// directly: a caller who sent {"archives": true} and omitted the
// other switch turned delete-on-removal OFF without asking, and an
// empty body turned both off and answered 200. That is dc748de1's
// shape — a partial write clearing what it was not told about —
// which repo.update already fixed this way and this op did not.
//
// Found by probing whether a bucket can be bound through the API
// at all: the probe was an empty POST here, and it succeeded.
Archives *bool `json:"archives"`
DeleteObjects *bool `json:"deleteObjects"`
}
// mayRemove refuses a removal that is neither the uploader's own nor a
// moderator's.
//
// THE HOLE IT CLOSES (G-147, found by jva's falsifier the hour the
// attachment surface became testable from a non-admin vantage): the
// attach letters b and B are in the READER preset, because attaching
// is something a reader does — fossil's model, deliberately. This op
// took those same letters and nothing else, so every reader on the
// forge could permanently remove ANY attachment, including an
// administrator's vouched one. Its sibling `reject` was gated on the
// moderation letters all along, so a reader was refused the moderation
// op and handed its destructive half.
//
// The letters alone cannot decide this, which is why the check is here
// rather than in Caps: removing your OWN upload is part of attaching,
// and removing SOMEBODY ELSE'S is moderation. The distinction is the
// A card's author, so the row has to be read before the answer exists.
//
// An anonymous caller matches nobody: an empty author beside an empty
// row user is two absences, not one person.
func (d Deps) mayRemove(ctx context.Context, repo *Repo, name string, f attachFamily, filename string) error {
if d.holds(ctx, name, f.vouchLetter()+"a") {
return nil
}
if f.Local {
// The forge tier answers who uploaded it; the ownership rule is
// the same one.
held, err := d.localAttachmentRows(name, f.Target)
if err != nil {
return err
}
for _, row := range held {
if row.Name == filename {
return d.judgeRemoval(f, filename, row.User, author(ctx))
}
}
return nil
}
live, err := repo.Index.AttachmentsFor(f.Target)
if err != nil {
return err
}
who := author(ctx)
for _, row := range live {
if row.Name != filename {
continue
}
return d.judgeRemoval(f, filename, row.User, who)
}
// Not in the index. That is usually "no such attachment", and
// remove() says so. But it is ALSO what a file looks like between
// reaching the store and being folded, and in that window this
// gate used to fall through deciding nothing at all — the caller
// was neither allowed nor refused, they were simply unexamined.
// Ask the store before concluding anything.
if uploader, ok := d.storeUploader(ctx, repo, f.Target, filename); ok {
return d.judgeRemoval(f, filename, uploader, who)
}
return nil
}
// judgeRemoval is the ownership question itself, asked the same way
// whichever source answered who uploaded the file.
func (d Deps) judgeRemoval(f attachFamily, filename, uploader, who string) error {
if who != "" && uploader == who {
return nil
}
return fmt.Errorf("%w: %s was attached by %s; removing an attachment that is not yours needs the %s capability",
ops.ErrForbidden, filename, rowUser(uploader), f.vouchLetter())
}
// storeUploader asks the store who attached filename to target, for
// the window where the store is ahead of the index.
//
// A store that cannot answer leaves the caller where they were: this
// only ever ADDS a refusal the index could not reach, and never turns
// one into a pass.
func (d Deps) storeUploader(ctx context.Context, repo *Repo, target, filename string) (string, bool) {
if repo.Zit == nil {
return "", false
}
rows, err := repo.Zit.AttachList(ctx, target)
if err != nil {
return "", false
}
for _, row := range rows {
if row.Name == filename {
return row.User, true
}
}
return "", false
}
// rowUser names an attachment's author for a refusal, without turning
// an anonymous upload into a person called "".
func rowUser(user string) string {
if user == "" {
return "an anonymous visitor"
}
return user
}
// remove is the removal itself, shared by attachment.remove and
// attachment.reject: the sourceless A card, the hold released, and the
// object deleted when the repository's switch says so.
func (d Deps) remove(ctx context.Context, repo *Repo, name string, f attachFamily, filename string) (attachmentRemoveOut, error) {
if f.Local {
return d.removeLocal(ctx, repo, name, f, filename)
}
live, err := repo.Index.AttachmentsFor(f.Target)
if err != nil {
return attachmentRemoveOut{}, err
}
var row *index.Attachment
for i := range live {
if live[i].Name == filename {
row = &live[i]
break
}
}
if row == nil {
// The store may be holding it while the index catches up, and
// "nothing named X is attached" is a lie in that window: the
// file is there, it is a second away, and a retry works. Say
// which of the two it is (jva, measured on a fresh upload).
if _, ok := d.storeUploader(ctx, repo, f.Target, filename); ok {
return attachmentRemoveOut{}, fmt.Errorf(
"%s is attached to %s but not indexed yet; try again in a moment: %w",
filename, f.Target, ops.ErrUnavailable)
}
return attachmentRemoveOut{}, fmt.Errorf("nothing named %q is attached to %s: %w", filename, f.Target, ops.ErrNotFound)
}
artifact, err := repo.Zit.AttachRm(ctx, f.Target, filename, author(ctx))
if err != nil {
if errors.Is(err, zitcli.ErrNoEntry) {
return attachmentRemoveOut{}, fmt.Errorf("%v: %w", err, ops.ErrInvalid)
}
return attachmentRemoveOut{}, err
}
out := attachmentRemoveOut{Artifact: artifact, Target: f.Target, Name: filename, Removed: true}
if d.Attachments != nil {
if err := d.Attachments.ReleaseHold(name, row.Hash); err != nil {
log.Printf("attachments: %s: releasing hold on %s: %v", name, row.Hash, err)
}
}
if row.External && row.SHA256 != "" {
// Deletion is the steward's decision (RFC 0016 §Writing); the
// switch on the binding is where they made it. Another live
// stub naming the same bytes keeps the object: the key is the
// content, not the attachment.
if _, b, bucket, err := d.boundBucket(name); err == nil && b.DeleteObjects && !d.otherStubNames(repo, row.SHA256) {
if err := bucket.Delete(ctx, attachClient(), attach.Key(row.SHA256), time.Now()); err != nil {
log.Printf("attachments: %s: deleting %s: %v", name, attach.Key(row.SHA256), err)
} else {
out.Deleted = true
}
}
}
return out, nil
}
// removeLocal is remove for a forge-tier ticket: the held row goes, and
// nothing in the log moves because nothing in the log was written. The
// object follows the same rule as for a stub — the repository's
// deleteObjects switch, and only when no other attachment, held or
// live, names the same bytes.
func (d Deps) removeLocal(ctx context.Context, repo *Repo, name string, f attachFamily, filename string) (attachmentRemoveOut, error) {
held, err := d.localAttachmentRows(name, f.Target)
if err != nil {
return attachmentRemoveOut{}, err
}
var row *index.Attachment
for i := range held {
if held[i].Name == filename {
row = &held[i]
break
}
}
if row == nil {
return attachmentRemoveOut{}, fmt.Errorf("nothing named %q is attached to %s: %w", filename, f.Target, ops.ErrNotFound)
}
if err := d.Local.LocalAttachmentRemove(name, f.Target, filename); err != nil {
return attachmentRemoveOut{}, err
}
out := attachmentRemoveOut{Target: f.Target, Name: filename, Removed: true}
if _, b, bucket, err := d.boundBucket(name); err == nil && b.DeleteObjects && row.SHA256 != "" {
n, err := d.Local.LocalSHACount(name, row.SHA256)
stubs, serr := repo.Index.LiveStubCount(row.SHA256)
if err == nil && serr == nil && n == 0 && stubs == 0 {
if err := bucket.Delete(ctx, attachClient(), attach.Key(row.SHA256), time.Now()); err != nil {
log.Printf("attachments: %s: deleting %s: %v", name, attach.Key(row.SHA256), err)
} else {
out.Deleted = true
}
}
}
return out, nil
}
// otherStubNames reports whether a live stub other than the one being
// removed still names sha. Searched through the live rows of every
// target is not possible cheaply, so this asks the row store the one
// question it can answer: is any live attachment's src a stub with
// this hash. The removal just written is not yet folded, so it still
// counts as one; two or more means another does.
func (d Deps) otherStubNames(repo *Repo, sha string) bool {
n, err := repo.Index.LiveStubCount(sha)
return err == nil && n > 1
}
// commitUpload is attachment.commit's body, shared with the ops that
// write a new object and then attach what was pasted while it was
// being written: the ticket is checked against its author and its
// repository, the target resolved (the ticket's own, or the one a
// deferred ticket is finally given), the object HEADed, the stub and
// the A card written, and the held rule applied.
func (d Deps) commitUpload(ctx context.Context, repo *Repo, name, ticket, target string) (attachmentCommitOut, error) {
t, ok, err := d.Attachments.UploadTicket(ticket)
if err != nil {
return attachmentCommitOut{}, err
}
if !ok || t.Repo != name {
return attachmentCommitOut{}, fmt.Errorf("no such upload ticket; begin again: %w", ops.ErrNotFound)
}
if t.Who != author(ctx) {
return attachmentCommitOut{}, fmt.Errorf("%w: this ticket was begun by someone else", ops.ErrForbidden)
}
f, err := d.ticketTarget(ctx, repo, name, t, target)
if err != nil {
return attachmentCommitOut{}, err
}
_, _, bucket, err := d.boundBucket(name)
if err != nil {
return attachmentCommitOut{}, err
}
size, present, err := bucket.Head(ctx, attachClient(), attach.Key(t.SHA256), time.Now())
if err != nil {
return attachmentCommitOut{}, fmt.Errorf("the bucket could not be asked about the object (%v): %w", err, ops.ErrUnavailable)
}
if !present {
return attachmentCommitOut{}, fmt.Errorf("the object is not in the bucket; the upload did not finish: %w", ops.ErrInvalid)
}
if size != t.Size {
return attachmentCommitOut{}, fmt.Errorf("the object is %d bytes and the upload was declared as %d: %w", size, t.Size, ops.ErrInvalid)
}
if f.Local {
// A forge-tier ticket: the object is checked exactly as above,
// and the stub is held in the forge database instead of being
// written to the log — there is no A card to write until the
// ticket is promoted, and promotion writes them all. The held
// rule is applied then too, against the uploader's letters,
// because that is when the file first appears in the DAG.
if err := d.Local.LocalAttachmentAdd(oltp.LocalAttachment{
Repo: name, Ticket: f.Target, Name: t.Name, Size: t.Size, Type: t.Type, SHA256: t.SHA256, Who: author(ctx),
}); err != nil {
return attachmentCommitOut{}, err
}
if err := d.Attachments.DeleteUploadTicket(ticket); err != nil {
log.Printf("attachments: %s: spending ticket %s: %v", name, ticket, err)
}
return attachmentCommitOut{Name: t.Name, Size: t.Size, Type: t.Type, SHA256: t.SHA256, Local: true}, nil
}
if repo.Zit == nil {
return attachmentCommitOut{}, fmt.Errorf("this repository has no store to write to: %w", ops.ErrUnavailable)
}
stub := attach.Stub{SHA256: t.SHA256, Size: t.Size, Type: t.Type, Name: t.Name}
artifact, blob, err := repo.Zit.AttachAdd(ctx, f.Target, t.Name, t.Type, author(ctx), "", stub.Format())
if err != nil {
if errors.Is(err, zitcli.ErrNoEntry) {
return attachmentCommitOut{}, fmt.Errorf("%v: %w", err, ops.ErrInvalid)
}
return attachmentCommitOut{}, err
}
out := attachmentCommitOut{Artifact: artifact, Blob: blob, Name: t.Name, Size: t.Size, Type: t.Type, SHA256: t.SHA256}
// The held rule (RFC 0016 §Trust): trusted for the family
// means the moderation letter or the steward's own. A hold
// that cannot be recorded is logged and the file shows —
// the A card is already written and cannot be unwritten,
// and a hidden hold row would be a hold nobody can release.
if !d.holds(ctx, name, f.vouchLetter()+"a") {
if err := d.Attachments.HoldAttachment(oltp.Hold{Repo: name, Artifact: artifact, Target: f.Target, Name: t.Name, Who: author(ctx)}); err != nil {
log.Printf("attachments: %s: recording hold on %s: %v", name, artifact, err)
} else {
out.Held = true
}
}
if err := d.Attachments.DeleteUploadTicket(ticket); err != nil {
log.Printf("attachments: %s: spending ticket %s: %v", name, ticket, err)
}
return out, nil
}
// ticketTarget resolves where a ticket's file attaches. A ticket begun
// against an object attaches there whatever the caller says now; a
// deferred one needs a target, and the target's family must be the
// one the ticket's letter was checked against — a ticket begun for a
// forum post with B must not land on a wiki page with no b.
func (d Deps) ticketTarget(ctx context.Context, repo *Repo, name string, t oltp.UploadTicket, target string) (attachFamily, error) {
if t.Target != "" {
return d.attachTarget(ctx, repo, name, t.Target)
}
if target == "" {
return attachFamily{}, fmt.Errorf("this upload was begun without a target; name one to commit it: %w", ops.ErrInvalid)
}
f, err := d.attachTarget(ctx, repo, name, target)
if err != nil {
return f, err
}
if f.Family != t.Family {
return f, fmt.Errorf("this upload was begun for a %s and %s is a %s: %w", t.Family, target, f.Family, ops.ErrInvalid)
}
return f, nil
}
// deferredFamily is the family a ticket without a target may name: the
// three kinds a composer writes. A technote is edited on its own page,
// where the section's form and paste apply.
func deferredFamily(family string) (attachFamily, error) {
switch family {
case "ticket", "forum", "wiki":
return attachFamily{Family: family}, nil
}
return attachFamily{}, fmt.Errorf("an upload without a target needs a family of ticket, forum or wiki, not %q: %w", family, ops.ErrInvalid)
}
// maxComposerAttachments bounds the `attach` list a composer op takes:
// twenty files on one post is a gallery, and past it the list is a
// mistake. Checked in code as well as by the router's rules, since
// the web layer invokes the ops directly.
const maxComposerAttachments = 20
func checkAttachList(tickets []string) error {
if len(tickets) > maxComposerAttachments {
return fmt.Errorf("at most %d files can be attached while composing: %w", maxComposerAttachments, ops.ErrInvalid)
}
return nil
}
// stagedAttachNote is what a composer op answers for its tickets when
// the post itself was held for moderation: there is no object to attach
// to, and the tickets keep for their hour, so the files are uploaded
// again with the post once it is released.
func stagedAttachNote(tickets []string) []string {
if len(tickets) == 0 {
return nil
}
return []string{fmt.Sprintf("the post is held for moderation; %d upload(s) were not attached", len(tickets))}
}
// attachTickets commits the tickets a composer collected against the
// object it just wrote. The object is already in the DAG, so nothing
// here can roll it back: each refusal is reported in words and the
// rest go on. nil when every ticket landed.
func (d Deps) attachTickets(ctx context.Context, repo *Repo, name, target string, tickets []string) []string {
if len(tickets) == 0 {
return nil
}
if d.Attachments == nil {
return []string{"this forge takes no uploads; nothing was attached"}
}
var errs []string
for _, tk := range tickets {
if _, err := d.commitUpload(ctx, repo, name, tk, target); err != nil {
errs = append(errs, tk+": "+err.Error())
}
}
return errs
}
// registerAttachWrite mounts the Phase 3 ops. Registered whether or not
// a store is wired, like the bucket ops, so the documented surface is
// the same on every forge and an unwired one answers unavailable.
func registerAttachWrite(r *ops.Registry, d Deps) error {
unwired := func() error {
if d.Attachments == nil {
return fmt.Errorf("this forge takes no uploads: %w", ops.ErrUnavailable)
}
return nil
}
if err := ops.Register(r, ops.Op[attachmentBeginIn, attachmentBeginOut]{
Name: "attachment.begin",
// Either upload letter reaches the surface; which one the
// target needs is decided inside, once the target's family is
// known (b for tickets, wiki pages and technotes, B for posts).
Caps: "bB",
Doc: "Begin an upload (RFC 0016 Phase 3): name the target, the file's name, size, type and SHA-256, and receive a presigned PUT — valid fifteen minutes, its content length and type signed — and a ticket for `attachment.commit`. Refused when the target is unknown or read-only, no bucket is bound, the size is over the binding's limit, or the type is not on the policy (text, the common images, PDF, JSON, the common audio and video; archives only when the repository allows them; executables never). An empty `target` with a `family` (ticket, forum or wiki) DEFERS the ticket for an object that does not exist yet or is about to change: the same gates apply against the family's letter, and the ticket is named its target by the `attach` list of `forum.post`, `forum.reply`, `ticket.new`, `ticket.set` and `wiki.save`, or by `attachment.commit`.",
Kind: ops.Command,
Auth: ops.Token,
// Under attachment/, singular, like attachment.head: the plural
// prefix is the multipart upload receiver's, mounted on the
// outer mux with a trailing wildcard, and it swallowed these
// two routes before the API router ever saw them.
Route: "POST /api/v1/r/{repo}/attachment/begin",
Run: func(ctx context.Context, in attachmentBeginIn) (attachmentBeginOut, error) {
if err := unwired(); err != nil {
return attachmentBeginOut{}, err
}
repo, err := d.attachWriteRepo(ctx, in.Repo)
if err != nil {
return attachmentBeginOut{}, err
}
var f attachFamily
if in.Target == "" {
// Deferred: gated on the family's letter now, on the
// target's existence and family at commit.
f, err = deferredFamily(in.Family)
if err != nil {
return attachmentBeginOut{}, err
}
if !d.holds(ctx, in.Repo, f.uploadLetter()) {
return attachmentBeginOut{}, fmt.Errorf("%w: attaching to a %s needs the %s capability", ops.ErrForbidden, f.Family, f.uploadLetter())
}
} else if f, err = d.attachTarget(ctx, repo, in.Repo, in.Target); err != nil {
return attachmentBeginOut{}, err
}
name := cleanAttachName(in.Name)
if name == "" {
return attachmentBeginOut{}, fmt.Errorf("the attachment needs a filename: %w", ops.ErrInvalid)
}
sha := strings.ToLower(in.SHA256)
if strings.Trim(sha, "0123456789abcdef") != "" {
return attachmentBeginOut{}, fmt.Errorf("sha256 must be 64 hex digits: %w", ops.ErrInvalid)
}
_, b, bucket, err := d.boundBucket(in.Repo)
if err != nil {
return attachmentBeginOut{}, err
}
if in.Size > b.LimitBytes {
return attachmentBeginOut{}, fmt.Errorf("%d bytes is over this repository's limit of %d: %w", in.Size, b.LimitBytes, ops.ErrInvalid)
}
ctype, _, err := mime.ParseMediaType(in.Type)
if err != nil || !attach.Allowed(ctype, b.Archives) {
return attachmentBeginOut{}, fmt.Errorf("files of type %q cannot be attached here: %w", in.Type, ops.ErrInvalid)
}
now := time.Now()
put, err := bucket.Presign(http.MethodPut, attach.Key(sha), uploadExpiry, now, map[string]string{
"content-length": fmt.Sprint(in.Size),
"content-type": ctype,
})
if err != nil {
return attachmentBeginOut{}, err
}
ticket, err := d.Attachments.PutUploadTicket(oltp.UploadTicket{
Repo: in.Repo, Target: f.Target, Family: f.Family, Name: name, Size: in.Size, Type: ctype, SHA256: sha, Who: author(ctx)})
if err != nil {
return attachmentBeginOut{}, err
}
return attachmentBeginOut{
Ticket: ticket, URL: put.String(), Method: http.MethodPut,
Headers: map[string]string{"Content-Length": fmt.Sprint(in.Size), "Content-Type": ctype},
Expires: now.Add(uploadExpiry).UTC().Format(time.RFC3339),
}, nil
},
}); err != nil {
return err
}
if err := ops.Register(r, ops.Op[attachmentCommitIn, attachmentCommitOut]{
Name: "attachment.commit",
Caps: "bB",
Doc: "Finish an upload: quarry HEADs the object and refuses if it is absent or its length differs from what `attachment.begin` was told, then writes the stub and the A card through zit and answers both hashes. Against a ticket on the forge tier the file is HELD instead (`local` is true, no artifact or blob yet) and its A card is written when the ticket is promoted. A deferred ticket (begun with no target) needs `target` here, checked against the family it was begun for. `held` is true when the caller lacks the family's moderation letter (q on tickets, l on the wiki and notes, 5 on the forum) and is not an administrator: the file is in the DAG and shown as a link until a moderator vouches for it.",
Kind: ops.Command,
Auth: ops.Token,
Route: "POST /api/v1/r/{repo}/attachment/commit",
Run: func(ctx context.Context, in attachmentCommitIn) (attachmentCommitOut, error) {
if err := unwired(); err != nil {
return attachmentCommitOut{}, err
}
repo, err := d.attachWriteRepo(ctx, in.Repo)
if err != nil {
return attachmentCommitOut{}, err
}
return d.commitUpload(ctx, repo, in.Repo, in.Ticket, in.Target)
},
}); err != nil {
return err
}
if err := ops.Register(r, ops.Op[attachmentRemoveIn, attachmentRemoveOut]{
Name: "attachment.remove",
Caps: "bB",
Doc: "Remove an attachment: an A card with no source, fossil's own deletion form. The attach letters reach this op because removing your OWN upload is part of attaching; removing anybody else's needs the family's moderation letter (q on tickets, l on the wiki and notes, 5 on the forum) or the administrator's. Nothing leaves the DAG. When the repository's setting `deleteObjects` is on and the attachment was an upload, the object is deleted from the bucket too, unless another live attachment names the same bytes.",
Kind: ops.Command,
Auth: ops.Token,
Route: "POST /api/v1/r/{repo}/attachments/remove",
Run: func(ctx context.Context, in attachmentRemoveIn) (attachmentRemoveOut, error) {
repo, err := d.attachWriteRepo(ctx, in.Repo)
if err != nil {
return attachmentRemoveOut{}, err
}
f, err := d.attachTarget(ctx, repo, in.Repo, in.Target)
if err != nil {
return attachmentRemoveOut{}, err
}
if err := d.mayRemove(ctx, repo, in.Repo, f, in.Name); err != nil {
return attachmentRemoveOut{}, err
}
return d.remove(ctx, repo, in.Repo, f, in.Name)
},
}); err != nil {
return err
}
resolveHeld := func(ctx context.Context, name, artifact string) (*Repo, index.Attachment, attachFamily, error) {
if err := unwired(); err != nil {
return nil, index.Attachment{}, attachFamily{}, err
}
repo, err := d.attachWriteRepo(ctx, name)
if err != nil {
return nil, index.Attachment{}, attachFamily{}, err
}
row, ok, err := repo.Index.AttachmentByHash(artifact)
if err != nil {
return nil, index.Attachment{}, attachFamily{}, err
}
if !ok {
return nil, index.Attachment{}, attachFamily{}, fmt.Errorf("no attachment artifact %q: %w", artifact, ops.ErrNotFound)
}
f, ok, err := attachFamilyOf(repo.Index, row.Target)
if err != nil {
return nil, index.Attachment{}, attachFamily{}, err
}
if !ok {
return nil, index.Attachment{}, attachFamily{}, fmt.Errorf("the attachment's target %q is unknown: %w", row.Target, ops.ErrNotFound)
}
if !d.holds(ctx, name, f.vouchLetter()+"a") {
return nil, index.Attachment{}, attachFamily{}, fmt.Errorf("%w: reviewing a %s attachment needs the %s capability", ops.ErrForbidden, f.Family, f.vouchLetter())
}
return repo, row, f, nil
}
if err := ops.Register(r, ops.Op[attachmentVouchIn, attachmentVouchOut]{
Name: "attachment.vouch",
Caps: "ql5a",
Doc: "Vouch for a held upload: it renders from now on. Needs the family's moderation letter (q, l or 5) or the administrator's.",
Kind: ops.Command,
Auth: ops.Token,
Route: "POST /api/v1/r/{repo}/attachments/vouch",
Run: func(ctx context.Context, in attachmentVouchIn) (attachmentVouchOut, error) {
_, row, _, err := resolveHeld(ctx, in.Repo, in.Artifact)
if err != nil {
return attachmentVouchOut{}, err
}
if err := d.Attachments.ReleaseHold(in.Repo, row.Hash); err != nil {
return attachmentVouchOut{}, err
}
d.logModeration(in.Repo, author(ctx), "attach-vouch", row.Name, row.Hash, in.Reason, row.User)
// And tell the uploader, for the same reason an approved
// forum post tells its author: they should not have to go
// back and look to find out.
//
// This was missed when rejection was wired, and found by
// counting the write paths that resolve a held thing rather
// than the ones I remembered — three of them, and only two
// told anybody. A forum post being approved told its author
// while an attachment being approved told nobody, which is
// the same asymmetry jva caught pointing the other way.
d.notifyModerated(in.Repo, row.User, author(ctx), "attachment", "approved",
row.Name, in.Reason, d.link("/r/"+in.Repo+"/attachment/"+row.Src+"/get"),
"attachment:"+row.Hash)
return attachmentVouchOut{Artifact: row.Hash}, nil
},
}); err != nil {
return err
}
if err := ops.Register(r, ops.Op[attachmentVouchIn, attachmentRemoveOut]{
Name: "attachment.reject",
Caps: "ql5a",
Doc: "Reject a held upload: the hold is cleared and the attachment removed as `attachment.remove` would, object deletion included when the repository says so.",
Kind: ops.Command,
Auth: ops.Token,
Route: "POST /api/v1/r/{repo}/attachments/reject",
Run: func(ctx context.Context, in attachmentVouchIn) (attachmentRemoveOut, error) {
repo, row, f, err := resolveHeld(ctx, in.Repo, in.Artifact)
if err != nil {
return attachmentRemoveOut{}, err
}
if err := d.Attachments.ReleaseHold(in.Repo, row.Hash); err != nil {
return attachmentRemoveOut{}, err
}
// Recorded BEFORE the removal, because the removal is what
// makes the file unfindable: a log line written afterwards
// would be the only trace, and if the remove failed halfway
// the log would claim something that did not happen. This
// order can leave a line for a rejection that then errors,
// which is the harmless direction — the hold is already
// released either way, and a moderator can see what they
// attempted.
d.logModeration(in.Repo, author(ctx), "attach-reject", row.Name, row.Hash, in.Reason, row.User)
// The uploader is the one person this concerns and, until
// now, the only one never told: the file is about to be
// removed, so there is nothing left for them to find and no
// state change for them to notice. row.User is who attached
// it, carried on the A card (b32aeba12bc6).
d.notifyModerated(in.Repo, row.User, author(ctx), "attachment", "rejected",
row.Name, in.Reason, "", "attachment:"+row.Hash)
return d.remove(ctx, repo, in.Repo, f, row.Name)
},
}); err != nil {
return err
}
if err := ops.Register(r, ops.Op[attachmentHeldIn, attachmentHeldOut]{
Name: "attachment.held",
Caps: "ql5a",
Doc: "The review queue: uploads on this repository nobody trusted has vouched for, oldest first.",
Kind: ops.Query,
Auth: ops.Token,
Route: "GET /api/v1/r/{repo}/attachments/held",
Run: func(ctx context.Context, in attachmentHeldIn) (attachmentHeldOut, error) {
if err := unwired(); err != nil {
return attachmentHeldOut{}, err
}
if _, err := d.repoFor(ctx, in.Repo); err != nil {
return attachmentHeldOut{}, err
}
holds, err := d.Attachments.Holds(in.Repo)
if err != nil {
return attachmentHeldOut{}, err
}
if holds == nil {
holds = []oltp.Hold{}
}
return attachmentHeldOut{Holds: holds}, nil
},
}); err != nil {
return err
}
if err := ops.Register(r, ops.Op[attachmentLinkIn, attachmentLinkOut]{
Name: "attachment.link",
Caps: "jr",
Doc: "Where an uploaded attachment's bytes are: a link on the bucket's public base when the repository is public and one is set, otherwise a presigned GET good for an hour that names the file and its type. `inline` is answered only for a vouched file whose type may render in a page; SVG never. A real (non-upload) attachment answers `external: false` and is served by quarry's own routes.",
Kind: ops.Query,
Auth: ops.Anonymous,
Route: "GET /api/v1/r/{repo}/attachment/{hash}/link",
Run: func(ctx context.Context, in attachmentLinkIn) (attachmentLinkOut, error) {
repo, err := d.repoFor(ctx, in.Repo)
if err != nil {
return attachmentLinkOut{}, err
}
row, ok, err := repo.Index.AttachmentBySrc(in.Hash)
if err != nil {
return attachmentLinkOut{}, err
}
if !ok {
// A file held against a forge-tier ticket: in the
// bucket like any upload, addressed by its object key.
row, ok = d.localAttachmentBySrc(in.Repo, in.Hash)
}
if !ok {
return attachmentLinkOut{}, fmt.Errorf("no attachment %q: %w", in.Hash, ops.ErrNotFound)
}
out := attachmentLinkOut{External: row.External, Name: row.Name}
if !row.External {
return out, nil
}
out.Type, out.Size = row.Mimetype, row.Size
out.Held = d.withHolds(in.Repo, []index.Attachment{row})[0].Held
info, _, bucket, err := d.boundBucket(in.Repo)
if err != nil {
return attachmentLinkOut{}, fmt.Errorf("%s has no bucket bound, so this file cannot be reached: %w", in.Repo, ops.ErrNotFound)
}
inline := !out.Held && attach.Inline(row.Mimetype)
private := d.Private != nil && d.Private()[in.Repo]
if info.PublicBase != "" && !private {
out.Download = info.PublicBase + "/" + attach.Key(row.SHA256)
if inline {
out.Inline = out.Download
}
return out, nil
}
now := time.Now()
dl, err := bucket.PresignGet(attach.Key(row.SHA256), linkExpiry, now, row.Name, row.Mimetype, false)
if err != nil {
return attachmentLinkOut{}, err
}
out.Download = dl.String()
out.Expires = now.Add(linkExpiry).UTC().Format(time.RFC3339)
if inline {
il, err := bucket.PresignGet(attach.Key(row.SHA256), linkExpiry, now, row.Name, row.Mimetype, true)
if err != nil {
return attachmentLinkOut{}, err
}
out.Inline = il.String()
}
return out, nil
},
}); err != nil {
return err
}
if err := ops.Register(r, ops.Op[bucketLimitIn, bucketLimitOut]{
Name: "bucket.limit",
Caps: "bB",
Doc: "Whether this repository takes uploads and how large one may be — the uploader's question, answered to holders of the upload letters without the binding's detail.",
Kind: ops.Query,
Auth: ops.Token,
Route: "GET /api/v1/r/{repo}/bucket/limit",
Run: func(ctx context.Context, in bucketLimitIn) (bucketLimitOut, error) {
if _, err := d.repoFor(ctx, in.Repo); err != nil {
return bucketLimitOut{}, err
}
if d.Buckets == nil {
return bucketLimitOut{Repo: in.Repo}, nil
}
info, b, err := d.bucketOf(in.Repo)
if err != nil {
return bucketLimitOut{}, err
}
// Bound is the uploader's answer straight from bucketOf: a
// repository inheriting the forge's binding is reported
// unbound there, with the reason, so this op no longer
// re-derives "bound and not forge" — the reading bucket.get
// used to leave to each caller (G-130).
out := bucketLimitOut{Repo: in.Repo, Bound: info.Bound, Note: info.Note}
if out.Bound {
out.Limit = b.LimitBytes
}
return out, nil
},
}); err != nil {
return err
}
return ops.Register(r, ops.Op[bucketPolicyIn, BucketInfo]{
Name: "bucket.policy",
Caps: "a",
Doc: "The binding's two switches: `archives` opens the upload type policy to zip, gzip and tar; `deleteObjects` makes a removal delete the object from the bucket as well. Both are off by default.",
Kind: ops.Command,
Auth: ops.Token,
Route: "POST /api/v1/r/{repo}/bucket/policy",
Run: func(ctx context.Context, in bucketPolicyIn) (BucketInfo, error) {
if err := unwired(); err != nil {
return BucketInfo{}, err
}
if _, err := d.repoFor(ctx, in.Repo); err != nil {
return BucketInfo{}, err
}
if parent := d.companionParent(in.Repo); parent != "" {
return BucketInfo{}, fmt.Errorf("%s uses %s's bucket; set its policy there: %w", in.Repo, parent, ops.ErrInvalid)
}
// Read, then apply only what was named, the way
// repo.update does. Omitted means unchanged.
cur, _, cerr := d.bucketOf(in.Repo)
if cerr != nil {
return BucketInfo{}, cerr
}
archives, deleteObjects := cur.Archives, cur.DeleteObjects
if in.Archives != nil {
archives = *in.Archives
}
if in.DeleteObjects != nil {
deleteObjects = *in.DeleteObjects
}
if err := d.Attachments.SetBucketPolicy(in.Repo, archives, deleteObjects); err != nil {
return BucketInfo{}, fmt.Errorf("%v: %w", err, ops.ErrInvalid)
}
what := "attachments policy: archives " + onOff(archives) + ", delete objects on removal " + onOff(deleteObjects)
d.bucketSettingsEvent(ctx, in.Repo, what)
info, _, err := d.bucketOf(in.Repo)
return info, err
},
})
}
// uploadStatus is the router's own mapping of an op error to a status,
// for the one handler here that is not an op.
func uploadStatus(err error) int {
switch {
case errors.Is(err, ops.ErrNotFound):
return http.StatusNotFound
case errors.Is(err, ops.ErrInvalid):
return http.StatusBadRequest
case errors.Is(err, ops.ErrForbidden):
return http.StatusForbidden
case errors.Is(err, ops.ErrUnavailable):
return http.StatusServiceUnavailable
}
return http.StatusInternalServerError
}
func onOff(b bool) string {
if b {
return "on"
}
return "off"
}
// UploadForm is the no-script path (RFC 0016 §Writing): a multipart
// form with `file` (and an optional `name`) that quarry carries to the
// bucket on the reader's behalf — begin, the PUT, commit — through the
// same ops the browser would call, so every gate is the same gate.
//
// The file is held in memory while it is counted and hashed, never on
// disk, and bounded twice: the binding's limit, refused as soon as one
// byte past it arrives, and maxFormUploadBytes, past which the scripted
// path is the way. ctx must carry the caller the way a gated request's
// does; the ops refuse on the letters as they would for any client.
func UploadForm(ctx context.Context, d Deps, r *ops.Registry, repo, target string, req *http.Request) (attachmentCommitOut, error) {
info, b, err := d.bucketOf(repo)
if d.Buckets == nil || err != nil || !info.Bound {
if err == nil && info.Note != "" {
return attachmentCommitOut{}, fmt.Errorf("%s: %w", info.Note, ops.ErrInvalid)
}
return attachmentCommitOut{}, fmt.Errorf("%s has no bucket bound for uploads: %w", repo, ops.ErrInvalid)
}
limit := min(b.LimitBytes, maxFormUploadBytes)
mr, err := req.MultipartReader()
if err != nil {
return attachmentCommitOut{}, fmt.Errorf("expected a multipart form with a file: %w", ops.ErrInvalid)
}
var body bytes.Buffer
var name, ctype string
sum := sha256.New()
got := false
for {
part, err := mr.NextPart()
if errors.Is(err, io.EOF) {
break
}
if err != nil {
return attachmentCommitOut{}, fmt.Errorf("%v: %w", err, ops.ErrInvalid)
}
switch part.FormName() {
case "name":
text, _ := io.ReadAll(io.LimitReader(part, 512))
if n := cleanAttachName(string(text)); n != "" {
name = n
}
case "file":
if name == "" {
name = cleanAttachName(part.FileName())
}
ctype = part.Header.Get("Content-Type")
n, err := io.Copy(io.MultiWriter(&body, sum), io.LimitReader(part, limit+1))
if err != nil {
return attachmentCommitOut{}, err
}
if n > limit {
if limit < b.LimitBytes {
return attachmentCommitOut{}, fmt.Errorf("files over %d bytes need a browser with script, which sends them to the bucket directly: %w", limit, ops.ErrInvalid)
}
return attachmentCommitOut{}, fmt.Errorf("the file is over this repository's limit of %d bytes: %w", b.LimitBytes, ops.ErrInvalid)
}
got = true
}
part.Close()
}
if !got || name == "" {
return attachmentCommitOut{}, fmt.Errorf("choose a file: %w", ops.ErrInvalid)
}
if body.Len() == 0 {
return attachmentCommitOut{}, fmt.Errorf("the file is empty: %w", ops.ErrInvalid)
}
if ctype == "" || ctype == "application/octet-stream" {
// Browsers name a type from the extension and say octet-stream
// when they have none; the policy then refuses it, which is the
// right answer for a file nobody could name.
if guess := mime.TypeByExtension(strings.ToLower(path.Ext(name))); guess != "" {
ctype = guess
}
}
begin := func() (attachmentBeginOut, error) {
out, err := r.Invoke(ctx, "attachment.begin", attachmentBeginIn{
Repo: repo, Target: target, Name: name, Size: int64(body.Len()), Type: ctype,
SHA256: hex.EncodeToString(sum.Sum(nil))})
if err != nil {
return attachmentBeginOut{}, err
}
return out.(attachmentBeginOut), nil
}
send := func() (attachmentCommitOut, error) {
bo, err := begin()
if err != nil {
return attachmentCommitOut{}, err
}
put, err := http.NewRequestWithContext(ctx, bo.Method, bo.URL, bytes.NewReader(body.Bytes()))
if err != nil {
return attachmentCommitOut{}, err
}
put.ContentLength = int64(body.Len())
put.Header.Set("Content-Type", bo.Headers["Content-Type"])
resp, err := attachClient().Do(put)
if err != nil {
return attachmentCommitOut{}, fmt.Errorf("the bucket refused the upload: %w", ops.ErrUnavailable)
}
io.Copy(io.Discard, io.LimitReader(resp.Body, 1<<16))
resp.Body.Close()
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
return attachmentCommitOut{}, fmt.Errorf("the bucket answered %d to the upload: %w", resp.StatusCode, ops.ErrUnavailable)
}
out, err := r.Invoke(ctx, "attachment.commit", attachmentCommitIn{Repo: repo, Ticket: bo.Ticket})
if err != nil {
return attachmentCommitOut{}, err
}
return out.(attachmentCommitOut), nil
}
if d.Bulk == nil {
return send()
}
var out attachmentCommitOut
err = d.Bulk.DoNamed(ctx, "attachment", repo, func() error {
var err error
out, err = send()
return err
})
return out, err
}
// UploadReceiver is UploadForm on the API's own path, for a client
// with Basic credentials and no script: the same gates, authenticated
// the way every op under /api/ is.
// MountUploadReceiver mounts the multipart receiver AND re-opens the op
// routes it would otherwise swallow.
//
// The receiver takes a trailing wildcard — POST .../attachments/{target...}
// — because a target is a ticket id, a page name or a post hash and may
// carry slashes. That wildcard claims every literal name under the same
// prefix, and the ops mounted there are reached through the router at
// "/api/", a PREFIX match that loses to it. `attachment.begin` and
// `attachment.commit` escaped this by moving to the singular
// "/attachment/"; vouch, reject and remove did not, and answered
// "expected a multipart form with a file" to everyone, administrators
// included, so no API client could moderate an attachment at all
// (G-148, jva). The OpenAPI meanwhile documented them as JSON ops.
//
// Rather than remember which names are taken, the literal op routes are
// registered on the SAME mux, where ServeMux's own precedence settles
// it: a literal last segment matches a strict subset of the wildcard,
// so the op wins. Derived from the registry, so an op added under this
// prefix tomorrow is mounted the day it is written rather than the day
// somebody notices it answering 400.
func MountUploadReceiver(mux *http.ServeMux, d Deps, r *ops.Registry, verify Verify, capsFor CapsFor, router http.Handler) {
const claimed = "/api/v1/r/{repo}/attachments/"
mux.Handle("POST "+claimed+"{target...}", UploadReceiver(d, r, verify, capsFor))
if router == nil {
return
}
for _, op := range r.All() {
if op.Method != http.MethodPost || !strings.HasPrefix(op.Pattern, claimed) {
continue
}
if strings.ContainsAny(op.Pattern[len(claimed):], "{}/") {
continue // itself a wildcard: nothing to rescue
}
mux.Handle(op.Method+" "+op.Pattern, router)
}
}
func UploadReceiver(d Deps, r *ops.Registry, verify Verify, capsFor CapsFor) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
repo, target := req.PathValue("repo"), req.PathValue("target")
ctx := req.Context()
if verify != nil {
user, pass, _ := req.BasicAuth()
switch verify(repo, user, pass, ops.Token) {
case Denied:
w.Header().Set("WWW-Authenticate", `Basic realm="quarry"`)
writeError(w, http.StatusUnauthorized, errors.New("authentication required"))
return
case Insufficient:
writeError(w, http.StatusForbidden, errors.New("this account does not have the capability this request needs"))
return
}
ctx = ops.WithUser(ctx, user)
if capsFor != nil {
ctx = ops.WithCaps(ctx, capsFor(user, repo))
}
}
out, err := UploadForm(ctx, d, r, repo, target, req)
if err != nil {
writeError(w, uploadStatus(err), err)
return
}
writeJSON(w, http.StatusOK, out)
})
}