package s3
import (
"context"
"crypto/hmac"
"crypto/sha256"
"encoding/hex"
"fmt"
"io"
"net/http"
"net/http/httptest"
"net/url"
"os"
"path/filepath"
"sort"
"strconv"
"strings"
"sync"
"testing"
"time"
)
// metaBucket holds bytes, metadata and a disposition, answers HEAD
// with the metadata it was given, and lists two keys a page, URL-encoded
// when asked, so a listing must follow the continuation token and
// decode what it reads. Every request's SigV4 signature is recomputed
// here, independently of the client's code, from the secret and what
// actually arrived; a mismatch is S3's 403 SignatureDoesNotMatch. A
// metadata header outside the signature is refused, as S3 refuses an
// unsigned x-amz header on a presigned request.
type metaBucket struct {
// raw makes the bucket ignore encoding-type and send keys as they
// are, as a service that does not implement it would.
raw bool
mu sync.Mutex
objects map[string]metaObject
tokens []string
checked int
copies int
// copyFails answers a copy 200 with an error body, which S3 does
// when a copy fails after it has started answering.
copyFails bool
}
type metaObject struct {
body []byte
meta map[string]string
disposition string
ctype string
}
const testSecret = "SK"
// verifySigV4 is the query-string algorithm written out again from
// AWS's description, against the request as the server received it.
func verifySigV4(r *http.Request, secret string) bool {
q := r.URL.Query()
sig := q.Get("X-Amz-Signature")
cred := strings.SplitN(q.Get("X-Amz-Credential"), "/", 5) // key/date/region/s3/aws4_request
if sig == "" || len(cred) != 5 {
return false
}
enc := func(s string, slash bool) string {
var b strings.Builder
for _, c := range []byte(s) {
if c >= 'A' && c <= 'Z' || c >= 'a' && c <= 'z' || c >= '0' && c <= '9' || strings.IndexByte("-_.~", c) >= 0 || slash && c == '/' {
b.WriteByte(c)
} else {
fmt.Fprintf(&b, "%%%02X", c)
}
}
return b.String()
}
var params []string
for k, vs := range q {
if k != "X-Amz-Signature" {
params = append(params, enc(k, false)+"="+enc(vs[0], false))
}
}
sort.Strings(params)
signed := strings.Split(q.Get("X-Amz-SignedHeaders"), ";")
var canonHeaders strings.Builder
for _, h := range signed {
v := r.Header.Get(h)
switch h {
case "host":
v = r.Host
case "content-length":
v = strconv.FormatInt(r.ContentLength, 10)
}
canonHeaders.WriteString(h + ":" + strings.TrimSpace(v) + "\n")
}
canonical := strings.Join([]string{r.Method, enc(r.URL.Path, true), strings.Join(params, "&"),
canonHeaders.String(), q.Get("X-Amz-SignedHeaders"), "UNSIGNED-PAYLOAD"}, "\n")
hash := sha256.Sum256([]byte(canonical))
scope := strings.Join(cred[1:], "/")
toSign := "AWS4-HMAC-SHA256\n" + q.Get("X-Amz-Date") + "\n" + scope + "\n" + hex.EncodeToString(hash[:])
mac := func(k []byte, m string) []byte { h := hmac.New(sha256.New, k); h.Write([]byte(m)); return h.Sum(nil) }
k := mac([]byte("AWS4"+secret), cred[1])
k = mac(k, cred[2])
k = mac(k, cred[3])
k = mac(k, cred[4])
return hmac.Equal([]byte(hex.EncodeToString(mac(k, toSign))), []byte(sig))
}
func newMetaBucket(t *testing.T) (*httptest.Server, *metaBucket) {
t.Helper()
m := &metaBucket{objects: map[string]metaObject{}}
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
q := r.URL.Query()
if !verifySigV4(r, testSecret) {
w.WriteHeader(http.StatusForbidden)
fmt.Fprint(w, "<Error><Code>SignatureDoesNotMatch</Code></Error>")
return
}
signed := strings.Split(q.Get("X-Amz-SignedHeaders"), ";")
m.mu.Lock()
defer m.mu.Unlock()
m.checked++
if r.URL.Path == "/files" && r.Method == http.MethodGet {
if q.Get("list-type") != "2" {
w.WriteHeader(http.StatusBadRequest)
return
}
m.tokens = append(m.tokens, q.Get("continuation-token"))
var keys []string
for k := range m.objects {
if strings.HasPrefix(k, q.Get("prefix")) {
keys = append(keys, k)
}
}
sort.Strings(keys)
start, _ := strconv.Atoi(q.Get("continuation-token"))
end := min(start+2, len(keys))
fmt.Fprint(w, `<?xml version="1.0" encoding="UTF-8"?><ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">`)
encode := q.Get("encoding-type") == "url" && !m.raw
if encode {
fmt.Fprint(w, "<EncodingType>url</EncodingType>")
}
for _, k := range keys[start:end] {
shown := k
if encode {
shown = url.QueryEscape(k)
}
fmt.Fprintf(w, "<Contents><Key>%s</Key><Size>%d</Size></Contents>", shown, len(m.objects[k].body))
}
if end < len(keys) {
fmt.Fprintf(w, "<IsTruncated>true</IsTruncated><NextContinuationToken>%d</NextContinuationToken>", end)
} else {
fmt.Fprint(w, "<IsTruncated>false</IsTruncated>")
}
fmt.Fprint(w, "</ListBucketResult>")
return
}
key := strings.TrimPrefix(r.URL.Path, "/files/")
switch r.Method {
case http.MethodPut:
meta := map[string]string{}
for name := range r.Header {
lower := strings.ToLower(name)
if strings.HasPrefix(lower, "x-amz-meta-") {
if !containsString(signed, lower) {
w.WriteHeader(http.StatusForbidden)
return
}
meta[strings.TrimPrefix(lower, "x-amz-meta-")] = r.Header.Get(name)
}
}
if r.Header.Get("Content-Disposition") != "" && !containsString(signed, "content-disposition") {
w.WriteHeader(http.StatusForbidden)
return
}
body, _ := io.ReadAll(r.Body)
// CopyObject: the bytes come from the source, and with
// REPLACE the headers from this request. Both x-amz headers
// must be signed, as on any presigned request.
if src := r.Header.Get("x-amz-copy-source"); src != "" {
if !containsString(signed, "x-amz-copy-source") || !containsString(signed, "x-amz-metadata-directive") ||
r.Header.Get("x-amz-metadata-directive") != "REPLACE" {
w.WriteHeader(http.StatusForbidden)
return
}
from, err := url.PathUnescape(strings.TrimPrefix(src, "/files/"))
o, ok := m.objects[from]
if err != nil || !ok {
w.WriteHeader(http.StatusNotFound)
return
}
if m.copyFails {
fmt.Fprint(w, "<Error><Code>InternalError</Code><Message>We encountered an internal error.</Message></Error>")
return
}
body = o.body
m.copies++
}
m.objects[key] = metaObject{body: body, meta: meta, disposition: r.Header.Get("Content-Disposition"), ctype: r.Header.Get("Content-Type")}
case http.MethodGet:
o, ok := m.objects[key]
if !ok {
w.WriteHeader(http.StatusNotFound)
return
}
w.Write(o.body)
case http.MethodHead:
o, ok := m.objects[key]
if !ok {
w.WriteHeader(http.StatusNotFound)
return
}
for k, v := range o.meta {
w.Header().Set("x-amz-meta-"+k, v)
}
w.Header().Set("Content-Length", strconv.Itoa(len(o.body)))
case http.MethodDelete:
delete(m.objects, key)
w.WriteHeader(http.StatusNoContent)
}
}))
t.Cleanup(srv.Close)
return srv, m
}
func containsString(list []string, s string) bool {
for _, x := range list {
if x == s {
return true
}
}
return false
}
// PutWith signs the type, the disposition and the metadata into the
// request; Stat reads the metadata back with the size; Stat of an
// absent key is false, not an error. Every request passed an
// independent recomputation of its signature.
func TestPutWithThenStat(t *testing.T) {
srv, m := newMetaBucket(t)
b := Bucket{Endpoint: srv.URL, Name: "files", Region: "auto", AccessKey: "AK", Secret: testSecret}
ctx := context.Background()
err := b.PutWith(ctx, nil, "unversioned/releases/a b.html", []byte("12345"), PutOptions{
ContentType: "application/octet-stream", Disposition: "attachment; filename*=UTF-8''a%20b.html",
Meta: map[string]string{"Mtime": "1727740800", "hash": "c0ffee"}}, time.Now())
if err != nil {
t.Fatalf("FAIL: put: %v", err)
}
o := m.objects["unversioned/releases/a b.html"]
if string(o.body) != "12345" || o.meta["mtime"] != "1727740800" || o.meta["hash"] != "c0ffee" ||
o.ctype != "application/octet-stream" || o.disposition != "attachment; filename*=UTF-8''a%20b.html" {
t.Errorf("FAIL: stored %+v", o)
}
obj, ok, err := b.Stat(ctx, nil, "unversioned/releases/a b.html", time.Now())
if err != nil || !ok || obj.Size != 5 || obj.Meta["mtime"] != "1727740800" || obj.Meta["hash"] != "c0ffee" {
t.Errorf("FAIL: stat = %+v %v %v", obj, ok, err)
}
if _, ok, err := b.Stat(ctx, nil, "unversioned/none", time.Now()); ok || err != nil {
t.Errorf("FAIL: stat absent = %v %v", ok, err)
}
if m.checked != 3 {
t.Errorf("FAIL: %d requests verified, want 3", m.checked)
}
t.Log("ok: type, disposition and metadata are signed on the way in and read on the way out")
}
// PutFile streams a file from disk under the same signed headers
// PutWith sets, so an archive of hundreds of megabytes is uploaded
// without being held in memory.
func TestPutFileStreamsAFile(t *testing.T) {
srv, m := newMetaBucket(t)
b := Bucket{Endpoint: srv.URL, Name: "files", Region: "auto", AccessKey: "AK", Secret: testSecret}
path := filepath.Join(t.TempDir(), "a.tar.gz")
if err := os.WriteFile(path, []byte("tarball bytes"), 0o644); err != nil {
t.Fatal(err)
}
err := b.PutFile(context.Background(), nil, "archive/r/h.tar.gz", path, PutOptions{ContentType: "application/gzip"}, time.Now())
if err != nil {
t.Fatalf("FAIL: put: %v", err)
}
o := m.objects["archive/r/h.tar.gz"]
if string(o.body) != "tarball bytes" || o.ctype != "application/gzip" {
t.Fatalf("FAIL: stored %+v", o)
}
if m.checked != 1 {
t.Fatalf("FAIL: %d requests verified, want 1", m.checked)
}
t.Log("ok: a file goes up with its length and type signed")
}
// The verifier is not a rubber stamp: a request signed with another
// secret is refused, and the refusal reaches the caller.
func TestTheFakeBucketChecksSignatures(t *testing.T) {
srv, _ := newMetaBucket(t)
b := Bucket{Endpoint: srv.URL, Name: "files", Region: "auto", AccessKey: "AK", Secret: "wrong"}
err := b.PutWith(context.Background(), nil, "unversioned/a", []byte("x"), PutOptions{ContentType: "text/plain"}, time.Now())
if err == nil || !strings.Contains(err.Error(), "SignatureDoesNotMatch") {
t.Fatalf("FAIL: a wrong secret was accepted: %v", err)
}
t.Log("ok: a bad signature is refused")
}
// List follows the continuation token to the end, keeps to the prefix,
// asks for URL-encoded keys and decodes them, so a key with a space, a
// percent sign, a plus or a character XML cannot carry comes back as
// it was written.
func TestListPaginatesAndDecodes(t *testing.T) {
srv, m := newMetaBucket(t)
for _, k := range []string{"unversioned/a b", "unversioned/b%41", "unversioned/c+d", "unversioned/.tombstones/d", "unversioned/e\x01", "attachments/x"} {
m.objects[k] = metaObject{body: []byte(k)}
}
b := Bucket{Endpoint: srv.URL, Name: "files", Region: "auto", AccessKey: "AK", Secret: testSecret}
got, err := b.List(context.Background(), nil, "unversioned/", time.Now())
if err != nil {
t.Fatalf("FAIL: list: %v", err)
}
var keys []string
for _, o := range got {
keys = append(keys, o.Key)
if o.Size != int64(len(o.Key)) {
t.Errorf("FAIL: %q size %d", o.Key, o.Size)
}
}
want := []string{"unversioned/.tombstones/d", "unversioned/a b", "unversioned/b%41", "unversioned/c+d", "unversioned/e\x01"}
if strings.Join(keys, "|") != strings.Join(want, "|") {
t.Errorf("FAIL: keys = %q", keys)
}
if strings.Join(m.tokens, ",") != ",2,4" {
t.Errorf("FAIL: tokens = %q", m.tokens)
}
if m.checked != 3 {
t.Errorf("FAIL: %d pages verified, want 3", m.checked)
}
t.Log("ok: a listing pages through the continuation token and decodes its keys")
}
// A service that ignores encoding-type sends keys as they are and says
// nothing about encoding; they are taken as sent, so a "+" stays a "+".
func TestListTakesUnencodedKeysAsSent(t *testing.T) {
srv, m := newMetaBucket(t)
m.raw = true
m.objects["unversioned/c++.tgz"] = metaObject{body: []byte("x")}
b := Bucket{Endpoint: srv.URL, Name: "files", Region: "auto", AccessKey: "AK", Secret: testSecret}
got, err := b.List(context.Background(), nil, "unversioned/", time.Now())
if err != nil || len(got) != 1 || got[0].Key != "unversioned/c++.tgz" {
t.Fatalf("FAIL: list = %+v %v", got, err)
}
t.Log("ok: unencoded keys are not decoded")
}