// Package s3 is the smallest thing that speaks the S3 signature: a
// query-string presign (SigV4) and a probe that uses it. No SDK,
// because the SDK is a hundred packages for two operations, and
// because the forge never handles an ATTACHMENT's bytes (RFC 0016):
// every upload and download is a presigned request the browser makes
// to the steward's bucket, so what the forge needs is a URL, not a
// client.
//
// Put and Get are the exception, and it is narrow on purpose. Objects
// the FORGE owns — a repository's avatar, in the forge's own bucket —
// are small, have to be validated before they are stored, and must be
// SERVED by the forge rather than linked: quarry's privacy page
// promises the interface loads nothing from another host, and an
// <img> pointed at R2 would break that for every reader of every page
// it appears on. Those two both hold the bytes, which is why they
// exist and why nothing else here does.
//
// Path-style addressing throughout — <endpoint>/<bucket>/<key> — since
// it is what every S3-compatible endpoint accepts (R2 and MinIO
// included), where virtual-hosted style needs DNS the steward may not
// have arranged.
package s3
import (
"bytes"
"context"
"crypto/hmac"
"crypto/rand"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"regexp"
"sort"
"strings"
"time"
)
// Bucket is one binding's connection: where the bucket is and the
// key that may write to it.
type Bucket struct {
// Endpoint is the service URL with a scheme and host and nothing
// after them: https://<account>.r2.cloudflarestorage.com,
// https://s3.us-east-1.amazonaws.com, http://127.0.0.1:9000.
Endpoint string
Name string
// Region is what the provider expects in the credential scope;
// R2 says "auto".
Region string
AccessKey string
Secret string
}
const (
algorithm = "AWS4-HMAC-SHA256"
unsignedPayload = "UNSIGNED-PAYLOAD"
amzDate = "20060102T150405Z"
scopeDate = "20060102"
)
// Presign builds a URL that lets whoever holds it perform method on
// key until expires have passed, without holding the secret. Every
// header in signedHeaders is bound into the signature — the request
// must then carry exactly those values, which is how an upload's
// content-length and content-type are fixed by the forge rather than
// chosen by the browser.
func (b Bucket) Presign(method, key string, expires time.Duration, now time.Time, signedHeaders map[string]string) (*url.URL, error) {
return b.presign(method, key, expires, now, signedHeaders, nil)
}
// PresignGet is Presign for a download whose answer the forge shapes:
// S3's response-content-disposition and response-content-type ride
// in the signed query, so the bucket serves the stub's filename and
// declared type rather than whatever the object was PUT with. Both
// are signed, which is what stops a holder of the URL renaming the
// file or handing the browser a type of their own.
func (b Bucket) PresignGet(key string, expires time.Duration, now time.Time, filename, ctype string, inline bool) (*url.URL, error) {
disposition := "attachment"
if inline {
disposition = "inline"
}
extra := url.Values{}
extra.Set("response-content-disposition", disposition+"; filename*=UTF-8''"+url.PathEscape(filename))
if ctype != "" {
extra.Set("response-content-type", ctype)
}
return b.presign(http.MethodGet, key, expires, now, nil, extra)
}
func (b Bucket) presign(method, key string, expires time.Duration, now time.Time, signedHeaders map[string]string, extra url.Values) (*url.URL, error) {
base, err := url.Parse(b.Endpoint)
if err != nil || base.Host == "" || (base.Scheme != "http" && base.Scheme != "https") {
return nil, fmt.Errorf("s3: endpoint %q is not an http(s) url", b.Endpoint)
}
if b.Name == "" || key == "" {
return nil, errors.New("s3: a bucket and a key are needed")
}
path := "/" + uriEncode(b.Name, false) + "/" + uriEncode(key, false)
q := b.sign(method, base.Host, path, expires, now, signedHeaders, extra)
// RawPath carries the exact bytes that were signed; url.URL would
// otherwise re-encode the key by its own rules and the service
// would hash something else.
return &url.URL{Scheme: base.Scheme, Host: base.Host,
Path: "/" + b.Name + "/" + key, RawPath: path, RawQuery: q.Encode()}, nil
}
// sign is the SigV4 query-string algorithm over an already-canonical
// host and path, kept apart from Presign so it can be checked against
// AWS's published vector, which is virtual-hosted style.
func (b Bucket) sign(method, host, canonicalPath string, expires time.Duration, now time.Time, signedHeaders map[string]string, extra url.Values) url.Values {
now = now.UTC()
scope := now.Format(scopeDate) + "/" + b.Region + "/s3/aws4_request"
headers := map[string]string{"host": host}
for k, v := range signedHeaders {
headers[strings.ToLower(strings.TrimSpace(k))] = strings.TrimSpace(v)
}
names := make([]string, 0, len(headers))
for k := range headers {
names = append(names, k)
}
sort.Strings(names)
var canonHeaders strings.Builder
for _, k := range names {
canonHeaders.WriteString(k + ":" + headers[k] + "\n")
}
signed := strings.Join(names, ";")
q := url.Values{}
for k, vs := range extra {
for _, v := range vs {
q.Add(k, v)
}
}
q.Set("X-Amz-Algorithm", algorithm)
q.Set("X-Amz-Credential", b.AccessKey+"/"+scope)
q.Set("X-Amz-Date", now.Format(amzDate))
q.Set("X-Amz-Expires", fmt.Sprint(int64(expires.Seconds())))
q.Set("X-Amz-SignedHeaders", signed)
canonical := strings.Join([]string{
method, canonicalPath, canonicalQuery(q), canonHeaders.String(), signed, unsignedPayload,
}, "\n")
toSign := strings.Join([]string{
algorithm, now.Format(amzDate), scope, hexSHA256([]byte(canonical)),
}, "\n")
k := hmacSHA256([]byte("AWS4"+b.Secret), []byte(now.Format(scopeDate)))
k = hmacSHA256(k, []byte(b.Region))
k = hmacSHA256(k, []byte("s3"))
k = hmacSHA256(k, []byte("aws4_request"))
q.Set("X-Amz-Signature", hex.EncodeToString(hmacSHA256(k, []byte(toSign))))
return q
}
// canonicalQuery is the query string as SigV4 hashes it: sorted by
// name, every name and value encoded with the unreserved set only.
// url.Values.Encode keeps "/" bare and would spell the credential
// scope differently from what the service recomputes.
func canonicalQuery(q url.Values) string {
keys := make([]string, 0, len(q))
for k := range q {
keys = append(keys, k)
}
sort.Strings(keys)
parts := make([]string, 0, len(keys))
for _, k := range keys {
parts = append(parts, uriEncode(k, true)+"="+uriEncode(q.Get(k), true))
}
return strings.Join(parts, "&")
}
// uriEncode is AWS's own encoder: RFC 3986 unreserved characters
// stay, "/" stays in a path, and everything else is %XX in upper
// case. It is not url.PathEscape, which keeps characters S3 expects
// encoded.
func uriEncode(s string, encodeSlash bool) string {
var sb strings.Builder
for i := 0; i < len(s); i++ {
c := s[i]
switch {
case c >= 'A' && c <= 'Z', c >= 'a' && c <= 'z', c >= '0' && c <= '9',
c == '-', c == '_', c == '.', c == '~':
sb.WriteByte(c)
case c == '/' && !encodeSlash:
sb.WriteByte(c)
default:
fmt.Fprintf(&sb, "%%%02X", c)
}
}
return sb.String()
}
func hexSHA256(b []byte) string {
sum := sha256.Sum256(b)
return hex.EncodeToString(sum[:])
}
func hmacSHA256(key, msg []byte) []byte {
h := hmac.New(sha256.New, key)
h.Write(msg)
return h.Sum(nil)
}
// ProbeError is what a probe request came back with: the verb, the
// status, and the service's own <Code> when the body carried one.
// That is everything a steward needs to fix a binding and nothing
// that would let them, or a log line, learn the secret.
type ProbeError struct {
Verb string
Status int
Code string
}
func (e *ProbeError) Error() string {
if e.Code != "" {
return fmt.Sprintf("%s returned %d %s", e.Verb, e.Status, e.Code)
}
return fmt.Sprintf("%s returned %d", e.Verb, e.Status)
}
var codeRE = regexp.MustCompile(`<Code>([^<]{1,80})</Code>`)
// probeBody is what the probe writes. A handful of bytes: the point is
// the signature, not the transfer.
const probeBody = "quarry attachment probe\n"
// Probe proves a binding works the way an upload will: a presigned
// PUT of a few bytes under attachments/, then a presigned DELETE of
// the same object. Both verbs matter — a token scoped to read only
// fails the first, one scoped without delete fails the second and
// would leave every removal an orphan. Any non-2xx answer is an error
// naming the verb and the status.
func (b Bucket) Probe(ctx context.Context, client *http.Client, now time.Time) error {
if client == nil {
client = http.DefaultClient
}
key := "attachments/.probe-" + rand.Text()
body := probeBody
put, err := b.Presign(http.MethodPut, key, 5*time.Minute, now, map[string]string{
"content-length": fmt.Sprint(len(body)),
"content-type": "text/plain",
})
if err != nil {
return err
}
req, err := http.NewRequestWithContext(ctx, http.MethodPut, put.String(), strings.NewReader(body))
if err != nil {
return err
}
req.ContentLength = int64(len(body))
req.Header.Set("Content-Type", "text/plain")
if err := do(client, req); err != nil {
return err
}
del, err := b.Presign(http.MethodDelete, key, 5*time.Minute, now, nil)
if err != nil {
return err
}
req, err = http.NewRequestWithContext(ctx, http.MethodDelete, del.String(), nil)
if err != nil {
return err
}
return do(client, req)
}
// Put stores bytes the FORGE holds, rather than bytes a browser
// uploads: a presigned PUT the forge itself sends. Attachments go
// straight from the page to the bucket and never pass through here,
// which is the whole point of that design — but a repository's avatar
// is small, arrives on an ordinary form post, and has to be checked
// before it is stored, so the forge is already holding the bytes.
//
// Probe proves this exact path works at binding time.
func (b Bucket) Put(ctx context.Context, client *http.Client, key, ctype string, body []byte, now time.Time) error {
if client == nil {
client = http.DefaultClient
}
put, err := b.Presign(http.MethodPut, key, 5*time.Minute, now, map[string]string{
"content-length": fmt.Sprint(len(body)),
"content-type": ctype,
})
if err != nil {
return err
}
req, err := http.NewRequestWithContext(ctx, http.MethodPut, put.String(), bytes.NewReader(body))
if err != nil {
return err
}
req.ContentLength = int64(len(body))
req.Header.Set("Content-Type", ctype)
return do(client, req)
}
// Get reads an object back through the forge. The page never links a
// bucket URL directly: quarry's privacy page promises the interface
// loads nothing from another host, and an <img> pointed at R2 breaks
// that promise for every reader of every page the image appears on.
func (b Bucket) Get(ctx context.Context, client *http.Client, key string, now time.Time) ([]byte, string, error) {
if client == nil {
client = http.DefaultClient
}
u, err := b.Presign(http.MethodGet, key, 5*time.Minute, now, nil)
if err != nil {
return nil, "", err
}
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)
if err != nil {
return nil, "", err
}
resp, err := client.Do(req)
if err != nil {
return nil, "", err
}
defer resp.Body.Close()
if resp.StatusCode/100 != 2 {
return nil, "", fmt.Errorf("s3: GET %s: %s", key, resp.Status)
}
raw, err := io.ReadAll(io.LimitReader(resp.Body, maxForgeObject+1))
if err != nil {
return nil, "", err
}
if int64(len(raw)) > maxForgeObject {
return nil, "", fmt.Errorf("s3: GET %s: object larger than %d bytes", key, maxForgeObject)
}
return raw, resp.Header.Get("Content-Type"), nil
}
// maxForgeObject bounds what Get will read into memory. Forge-held
// objects are small by construction (an avatar); the limit is here so
// a bucket that answers with something else cannot become a way to
// make the forge allocate without bound.
const maxForgeObject = 4 << 20
// Preflight asks the bucket what a browser at origin will ask before
// the page's scripted upload: an OPTIONS on the object URL naming the
// origin, the PUT and the content-type header the upload sends. The
// bucket answers from its CORS configuration, not from the signature,
// so a bucket with no rule for the forge fails here while the forge's
// own presigned PUT (Probe) passes — and every browser upload would
// fail until the steward adds the rule. A refusal names what was
// missing so the settings page can say so.
func (b Bucket) Preflight(ctx context.Context, client *http.Client, origin string, now time.Time) error {
if client == nil {
client = http.DefaultClient
}
u, err := b.Presign(http.MethodPut, "attachments/.preflight", time.Minute, now, nil)
if err != nil {
return err
}
req, err := http.NewRequestWithContext(ctx, http.MethodOptions, u.String(), nil)
if err != nil {
return err
}
req.Header.Set("Origin", origin)
req.Header.Set("Access-Control-Request-Method", http.MethodPut)
req.Header.Set("Access-Control-Request-Headers", "content-type")
resp, err := client.Do(req)
if err != nil {
return fmt.Errorf("preflight: %w", redact(err, req.URL))
}
defer resp.Body.Close()
io.Copy(io.Discard, io.LimitReader(resp.Body, 1<<16))
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
return &ProbeError{Verb: "preflight", Status: resp.StatusCode}
}
allowed := resp.Header.Get("Access-Control-Allow-Origin")
if allowed != "*" && allowed != origin {
return fmt.Errorf("preflight allowed no origin for %s", origin)
}
if methods := resp.Header.Get("Access-Control-Allow-Methods"); methods != "" && !allowsPut(methods) {
return fmt.Errorf("preflight allows %s but not PUT", methods)
}
return nil
}
// allowsPut reads a comma-separated Access-Control-Allow-Methods.
func allowsPut(methods string) bool {
for _, m := range strings.Split(methods, ",") {
if m = strings.TrimSpace(m); strings.EqualFold(m, http.MethodPut) || m == "*" {
return true
}
}
return false
}
// Head asks whether key is in the bucket and how big it is. A 404 is
// the answer false, not an error: commit asks this of an object the
// browser may never have sent.
func (b Bucket) Head(ctx context.Context, client *http.Client, key string, now time.Time) (size int64, ok bool, err error) {
if client == nil {
client = http.DefaultClient
}
u, err := b.Presign(http.MethodHead, key, 5*time.Minute, now, nil)
if err != nil {
return 0, false, err
}
req, err := http.NewRequestWithContext(ctx, http.MethodHead, u.String(), nil)
if err != nil {
return 0, false, err
}
resp, err := client.Do(req)
if err != nil {
return 0, false, fmt.Errorf("HEAD: %w", redact(err, req.URL))
}
defer resp.Body.Close()
io.Copy(io.Discard, io.LimitReader(resp.Body, 1<<16))
switch {
case resp.StatusCode == http.StatusNotFound:
return 0, false, nil
case resp.StatusCode >= 200 && resp.StatusCode < 300:
return resp.ContentLength, true, nil
}
return 0, false, &ProbeError{Verb: http.MethodHead, Status: resp.StatusCode}
}
// Delete removes key. Deleting what is not there is a success at
// every S3, and here too.
func (b Bucket) Delete(ctx context.Context, client *http.Client, key string, now time.Time) error {
if client == nil {
client = http.DefaultClient
}
u, err := b.Presign(http.MethodDelete, key, 5*time.Minute, now, nil)
if err != nil {
return err
}
req, err := http.NewRequestWithContext(ctx, http.MethodDelete, u.String(), nil)
if err != nil {
return err
}
return do(client, req)
}
func do(client *http.Client, req *http.Request) error {
resp, err := client.Do(req)
if err != nil {
return fmt.Errorf("%s: %w", req.Method, redact(err, req.URL))
}
defer resp.Body.Close()
if resp.StatusCode >= 200 && resp.StatusCode < 300 {
io.Copy(io.Discard, io.LimitReader(resp.Body, 1<<16))
return nil
}
pe := &ProbeError{Verb: req.Method, Status: resp.StatusCode}
if body, err := io.ReadAll(io.LimitReader(resp.Body, 1<<16)); err == nil {
if m := codeRE.FindSubmatch(body); m != nil {
pe.Code = string(m[1])
}
}
return pe
}
// redact keeps a transport error's words and drops the URL it quotes,
// which carries the signature: not the secret, but derived from it
// and valid for a while.
func redact(err error, u *url.URL) error {
var ue *url.Error
if errors.As(err, &ue) {
return ue.Err
}
return errors.New(strings.ReplaceAll(err.Error(), u.String(), u.Host))
}