// Package s3 is the smallest thing that speaks the S3 signature: a
// query-string presign (SigV4) and a probe that uses it. No SDK,
// because the SDK is a hundred packages for two operations, and
// because the forge never handles the bytes (RFC 0016): every upload
// and download is a presigned request the browser makes to the
// steward's bucket, so what the forge needs is a URL, not a client.
//
// Path-style addressing throughout — <endpoint>/<bucket>/<key> — since
// it is what every S3-compatible endpoint accepts (R2 and MinIO
// included), where virtual-hosted style needs DNS the steward may not
// have arranged.
package s3
import (
"context"
"crypto/hmac"
"crypto/rand"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"regexp"
"sort"
"strings"
"time"
)
// Bucket is one binding's connection: where the bucket is and the
// key that may write to it.
type Bucket struct {
// Endpoint is the service URL with a scheme and host and nothing
// after them: https://<account>.r2.cloudflarestorage.com,
// https://s3.us-east-1.amazonaws.com, http://127.0.0.1:9000.
Endpoint string
Name string
// Region is what the provider expects in the credential scope;
// R2 says "auto".
Region string
AccessKey string
Secret string
}
const (
algorithm = "AWS4-HMAC-SHA256"
unsignedPayload = "UNSIGNED-PAYLOAD"
amzDate = "20060102T150405Z"
scopeDate = "20060102"
)
// Presign builds a URL that lets whoever holds it perform method on
// key until expires have passed, without holding the secret. Every
// header in signedHeaders is bound into the signature — the request
// must then carry exactly those values, which is how an upload's
// content-length and content-type are fixed by the forge rather than
// chosen by the browser.
func (b Bucket) Presign(method, key string, expires time.Duration, now time.Time, signedHeaders map[string]string) (*url.URL, error) {
return b.presign(method, key, expires, now, signedHeaders, nil)
}
// PresignGet is Presign for a download whose answer the forge shapes:
// S3's response-content-disposition and response-content-type ride
// in the signed query, so the bucket serves the stub's filename and
// declared type rather than whatever the object was PUT with. Both
// are signed, which is what stops a holder of the URL renaming the
// file or handing the browser a type of their own.
func (b Bucket) PresignGet(key string, expires time.Duration, now time.Time, filename, ctype string, inline bool) (*url.URL, error) {
disposition := "attachment"
if inline {
disposition = "inline"
}
extra := url.Values{}
extra.Set("response-content-disposition", disposition+"; filename*=UTF-8''"+url.PathEscape(filename))
if ctype != "" {
extra.Set("response-content-type", ctype)
}
return b.presign(http.MethodGet, key, expires, now, nil, extra)
}
func (b Bucket) presign(method, key string, expires time.Duration, now time.Time, signedHeaders map[string]string, extra url.Values) (*url.URL, error) {
base, err := url.Parse(b.Endpoint)
if err != nil || base.Host == "" || (base.Scheme != "http" && base.Scheme != "https") {
return nil, fmt.Errorf("s3: endpoint %q is not an http(s) url", b.Endpoint)
}
if b.Name == "" || key == "" {
return nil, errors.New("s3: a bucket and a key are needed")
}
path := "/" + uriEncode(b.Name, false) + "/" + uriEncode(key, false)
q := b.sign(method, base.Host, path, expires, now, signedHeaders, extra)
// RawPath carries the exact bytes that were signed; url.URL would
// otherwise re-encode the key by its own rules and the service
// would hash something else.
return &url.URL{Scheme: base.Scheme, Host: base.Host,
Path: "/" + b.Name + "/" + key, RawPath: path, RawQuery: q.Encode()}, nil
}
// sign is the SigV4 query-string algorithm over an already-canonical
// host and path, kept apart from Presign so it can be checked against
// AWS's published vector, which is virtual-hosted style.
func (b Bucket) sign(method, host, canonicalPath string, expires time.Duration, now time.Time, signedHeaders map[string]string, extra url.Values) url.Values {
now = now.UTC()
scope := now.Format(scopeDate) + "/" + b.Region + "/s3/aws4_request"
headers := map[string]string{"host": host}
for k, v := range signedHeaders {
headers[strings.ToLower(strings.TrimSpace(k))] = strings.TrimSpace(v)
}
names := make([]string, 0, len(headers))
for k := range headers {
names = append(names, k)
}
sort.Strings(names)
var canonHeaders strings.Builder
for _, k := range names {
canonHeaders.WriteString(k + ":" + headers[k] + "\n")
}
signed := strings.Join(names, ";")
q := url.Values{}
for k, vs := range extra {
for _, v := range vs {
q.Add(k, v)
}
}
q.Set("X-Amz-Algorithm", algorithm)
q.Set("X-Amz-Credential", b.AccessKey+"/"+scope)
q.Set("X-Amz-Date", now.Format(amzDate))
q.Set("X-Amz-Expires", fmt.Sprint(int64(expires.Seconds())))
q.Set("X-Amz-SignedHeaders", signed)
canonical := strings.Join([]string{
method, canonicalPath, canonicalQuery(q), canonHeaders.String(), signed, unsignedPayload,
}, "\n")
toSign := strings.Join([]string{
algorithm, now.Format(amzDate), scope, hexSHA256([]byte(canonical)),
}, "\n")
k := hmacSHA256([]byte("AWS4"+b.Secret), []byte(now.Format(scopeDate)))
k = hmacSHA256(k, []byte(b.Region))
k = hmacSHA256(k, []byte("s3"))
k = hmacSHA256(k, []byte("aws4_request"))
q.Set("X-Amz-Signature", hex.EncodeToString(hmacSHA256(k, []byte(toSign))))
return q
}
// canonicalQuery is the query string as SigV4 hashes it: sorted by
// name, every name and value encoded with the unreserved set only.
// url.Values.Encode keeps "/" bare and would spell the credential
// scope differently from what the service recomputes.
func canonicalQuery(q url.Values) string {
keys := make([]string, 0, len(q))
for k := range q {
keys = append(keys, k)
}
sort.Strings(keys)
parts := make([]string, 0, len(keys))
for _, k := range keys {
parts = append(parts, uriEncode(k, true)+"="+uriEncode(q.Get(k), true))
}
return strings.Join(parts, "&")
}
// uriEncode is AWS's own encoder: RFC 3986 unreserved characters
// stay, "/" stays in a path, and everything else is %XX in upper
// case. It is not url.PathEscape, which keeps characters S3 expects
// encoded.
func uriEncode(s string, encodeSlash bool) string {
var sb strings.Builder
for i := 0; i < len(s); i++ {
c := s[i]
switch {
case c >= 'A' && c <= 'Z', c >= 'a' && c <= 'z', c >= '0' && c <= '9',
c == '-', c == '_', c == '.', c == '~':
sb.WriteByte(c)
case c == '/' && !encodeSlash:
sb.WriteByte(c)
default:
fmt.Fprintf(&sb, "%%%02X", c)
}
}
return sb.String()
}
func hexSHA256(b []byte) string {
sum := sha256.Sum256(b)
return hex.EncodeToString(sum[:])
}
func hmacSHA256(key, msg []byte) []byte {
h := hmac.New(sha256.New, key)
h.Write(msg)
return h.Sum(nil)
}
// ProbeError is what a probe request came back with: the verb, the
// status, and the service's own <Code> when the body carried one.
// That is everything a steward needs to fix a binding and nothing
// that would let them, or a log line, learn the secret.
type ProbeError struct {
Verb string
Status int
Code string
}
func (e *ProbeError) Error() string {
if e.Code != "" {
return fmt.Sprintf("%s returned %d %s", e.Verb, e.Status, e.Code)
}
return fmt.Sprintf("%s returned %d", e.Verb, e.Status)
}
var codeRE = regexp.MustCompile(`<Code>([^<]{1,80})</Code>`)
// probeBody is what the probe writes. A handful of bytes: the point is
// the signature, not the transfer.
const probeBody = "quarry attachment probe\n"
// Probe proves a binding works the way an upload will: a presigned
// PUT of a few bytes under attachments/, then a presigned DELETE of
// the same object. Both verbs matter — a token scoped to read only
// fails the first, one scoped without delete fails the second and
// would leave every removal an orphan. Any non-2xx answer is an error
// naming the verb and the status.
func (b Bucket) Probe(ctx context.Context, client *http.Client, now time.Time) error {
if client == nil {
client = http.DefaultClient
}
key := "attachments/.probe-" + rand.Text()
body := probeBody
put, err := b.Presign(http.MethodPut, key, 5*time.Minute, now, map[string]string{
"content-length": fmt.Sprint(len(body)),
"content-type": "text/plain",
})
if err != nil {
return err
}
req, err := http.NewRequestWithContext(ctx, http.MethodPut, put.String(), strings.NewReader(body))
if err != nil {
return err
}
req.ContentLength = int64(len(body))
req.Header.Set("Content-Type", "text/plain")
if err := do(client, req); err != nil {
return err
}
del, err := b.Presign(http.MethodDelete, key, 5*time.Minute, now, nil)
if err != nil {
return err
}
req, err = http.NewRequestWithContext(ctx, http.MethodDelete, del.String(), nil)
if err != nil {
return err
}
return do(client, req)
}
// Head asks whether key is in the bucket and how big it is. A 404 is
// the answer false, not an error: commit asks this of an object the
// browser may never have sent.
func (b Bucket) Head(ctx context.Context, client *http.Client, key string, now time.Time) (size int64, ok bool, err error) {
if client == nil {
client = http.DefaultClient
}
u, err := b.Presign(http.MethodHead, key, 5*time.Minute, now, nil)
if err != nil {
return 0, false, err
}
req, err := http.NewRequestWithContext(ctx, http.MethodHead, u.String(), nil)
if err != nil {
return 0, false, err
}
resp, err := client.Do(req)
if err != nil {
return 0, false, fmt.Errorf("HEAD: %w", redact(err, req.URL))
}
defer resp.Body.Close()
io.Copy(io.Discard, io.LimitReader(resp.Body, 1<<16))
switch {
case resp.StatusCode == http.StatusNotFound:
return 0, false, nil
case resp.StatusCode >= 200 && resp.StatusCode < 300:
return resp.ContentLength, true, nil
}
return 0, false, &ProbeError{Verb: http.MethodHead, Status: resp.StatusCode}
}
// Delete removes key. Deleting what is not there is a success at
// every S3, and here too.
func (b Bucket) Delete(ctx context.Context, client *http.Client, key string, now time.Time) error {
if client == nil {
client = http.DefaultClient
}
u, err := b.Presign(http.MethodDelete, key, 5*time.Minute, now, nil)
if err != nil {
return err
}
req, err := http.NewRequestWithContext(ctx, http.MethodDelete, u.String(), nil)
if err != nil {
return err
}
return do(client, req)
}
func do(client *http.Client, req *http.Request) error {
resp, err := client.Do(req)
if err != nil {
return fmt.Errorf("%s: %w", req.Method, redact(err, req.URL))
}
defer resp.Body.Close()
if resp.StatusCode >= 200 && resp.StatusCode < 300 {
io.Copy(io.Discard, io.LimitReader(resp.Body, 1<<16))
return nil
}
pe := &ProbeError{Verb: req.Method, Status: resp.StatusCode}
if body, err := io.ReadAll(io.LimitReader(resp.Body, 1<<16)); err == nil {
if m := codeRE.FindSubmatch(body); m != nil {
pe.Code = string(m[1])
}
}
return pe
}
// redact keeps a transport error's words and drops the URL it quotes,
// which carries the signature: not the secret, but derived from it
// and valid for a while.
func redact(err error, u *url.URL) error {
var ue *url.Error
if errors.As(err, &ue) {
return ue.Err
}
return errors.New(strings.ReplaceAll(err.Error(), u.String(), u.Host))
}