// Package imgproxy serves the external images a rendered document names,
// from this forge instead of the reader's browser.
//
// A README can embed an image from anywhere: build badges, screenshots,
// diagrams. The forge's CSP admits images from itself only, so those
// rendered as broken, and loading them directly would tell every third
// party who reads the page. The renderer rewrites an external <img> to a
// signed /img URL; this package checks the signature, fetches the image
// once, caches it, and serves it.
//
// It must not become an open proxy or a way into the host's network:
//
// - Only URLs the renderer signed are fetched. The key lives beside the
// data it serves and never leaves the server.
// - Only http and https, and never a loopback, private, link-local or
// multicast address — checked at connect time on the address actually
// dialed, so a name that resolves inward is refused too.
// - Only image responses, capped in size and time.
// - SVG is served with a sandboxing CSP and nosniff, so an image cannot
// run script even if a browser is asked to open it directly.
//
// What makes it fast, in the order a reader feels it:
//
// - The URL is a PATH ending in the image's extension, not a query
// string. A CDN in front of the forge treats a query URL as dynamic
// and refuses to cache it; a path that looks like a file is cached at
// the edge, so the second reader in a city never reaches this server.
// - An ETag over the bytes, so a reader who has the image spends an
// empty 304 rather than the image again.
// - The response streams while it is fetched and cached, so the first
// byte does not wait for the last.
// - A stale image is served immediately and refreshed behind the
// reader, so nobody waits on the upstream after the first fetch.
// - The pixel size is recorded on the way past, so the renderer can
// write width and height into the tag and the page stops reflowing
// as images arrive.
package imgproxy
import (
"context"
"crypto/hmac"
"crypto/rand"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"image"
_ "image/gif" // DecodeConfig: the sizes readers actually embed
_ "image/jpeg" // …
_ "image/png" // …
"io"
"mime"
"net"
"net/http"
"net/url"
"os"
"path"
"path/filepath"
"regexp"
"sort"
"strconv"
"strings"
"sync"
"time"
)
const (
keyFile = "imageproxy.key"
cacheDir = "imgcache"
// MaxBytes caps one image. Badges are a few KiB; a screenshot in a
// README rarely passes a few hundred.
MaxBytes = 5 << 20
// TTL is how long a cached image is served before it is fetched
// again. Short, because badges are the common case and they change.
TTL = time.Hour
// StaleFor is how long past TTL a cached image is still served while
// a refresh runs behind the reader. A badge host that is slow, or
// down, then costs nobody a wait.
StaleFor = 7 * 24 * time.Hour
// EdgeTTL is what a CDN may keep. Longer than TTL because the edge
// revalidates against this server, which answers from its own cache.
EdgeTTL = 24 * time.Hour
// CacheBudget is what the cache may occupy on disk. Past it, the
// images fetched longest ago go first: a refresh only happens when
// somebody asks for an image, so the oldest fetch is the one nobody
// has wanted for longest.
CacheBudget = 512 << 20
// sweepEvery bounds how often the directory is walked, so a busy
// forge does not pay for a sweep on every fetch and an idle one pays
// for none at all.
sweepEvery = 10 * time.Minute
// tempAge is how long a temp file is left alone before it is read as
// abandoned, so a sweep never removes one a fetch is still writing.
tempAge = time.Hour
)
// Proxy is the signer and the handler.
type Proxy struct {
key []byte
dir string
client *http.Client
now func() time.Time
// budget is the disk the cache may hold; a field rather than the
// constant so a test can fill a small one.
budget int64
mu sync.Mutex // guards locks, refreshing and swept
locks map[string]*sync.Mutex
// swept is when the last sweep ran, so they stay rate limited.
swept time.Time
// refreshing names the URLs a background refresh is already running
// for, so a burst of readers on a stale image starts one fetch.
refreshing map[string]bool
}
// meta is what the cache remembers beside the bytes.
type meta struct {
Type string `json:"type"`
ETag string `json:"etag"`
Width int `json:"width,omitempty"`
Height int `json:"height,omitempty"`
Fetched string `json:"fetched"`
}
// New loads or creates the signing key under dataDir and prepares the
// cache beside it. allowPrivate admits private destinations; it exists
// for tests against a local server and is never set in production.
func New(dataDir string, allowPrivate bool) (*Proxy, error) {
key, err := loadKey(filepath.Join(dataDir, keyFile))
if err != nil {
return nil, err
}
dir := filepath.Join(dataDir, cacheDir)
if err := os.MkdirAll(dir, 0o750); err != nil {
return nil, err
}
return &Proxy{key: key, dir: dir, client: guardedClient(allowPrivate), now: time.Now,
budget: CacheBudget, locks: map[string]*sync.Mutex{}, refreshing: map[string]bool{}}, nil
}
func loadKey(path string) ([]byte, error) {
if b, err := os.ReadFile(path); err == nil && len(b) >= 32 {
return b, nil
}
b := make([]byte, 32)
if _, err := rand.Read(b); err != nil {
return nil, err
}
if err := os.WriteFile(path, b, 0o600); err != nil {
return nil, fmt.Errorf("imgproxy: key: %w", err)
}
return b, nil
}
func (p *Proxy) sign(u string) string {
m := hmac.New(sha256.New, p.key)
m.Write([]byte(u))
return base64.RawURLEncoding.EncodeToString(m.Sum(nil))
}
// cacheExts are the extensions a CDN caches without being configured to.
// An image whose URL ends in anything else is served under .img, which
// still carries the cache headers but may need an edge rule to be kept.
var cacheExts = map[string]bool{
".svg": true, ".png": true, ".jpg": true, ".jpeg": true, ".gif": true,
".webp": true, ".avif": true, ".ico": true, ".bmp": true,
}
// extOf is the extension the proxied path ends in, taken from the source
// URL's own path so a badge stays a .svg to anything looking at the URL.
func extOf(u *url.URL) string {
e := strings.ToLower(path.Ext(u.Path))
if cacheExts[e] {
return e
}
return ".img"
}
// URL is the proxied address for an external image, with the pixel size
// if this forge has fetched it before (0, 0 until then). It is "" for
// anything that is not an absolute http or https URL, which the caller
// leaves alone.
func (p *Proxy) URL(src string) (string, int, int) {
u, err := url.Parse(src)
if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" {
return "", 0, 0
}
// A path, not a query: a CDN caches a URL that looks like a file and
// treats one with a query string as dynamic.
proxied := "/img/" + p.sign(src) + "/" + base64.RawURLEncoding.EncodeToString([]byte(src)) + extOf(u)
if m, ok := p.meta(p.base(src)); ok {
return proxied, m.Width, m.Height
}
return proxied, 0, 0
}
func (p *Proxy) base(src string) string {
name := sha256.Sum256([]byte(src))
return filepath.Join(p.dir, hex.EncodeToString(name[:]))
}
// parse reads /img/<signature>/<encoded source><ext> back, and answers
// only for a signature this forge wrote.
func (p *Proxy) parse(reqPath string) (string, bool) {
rest := strings.TrimPrefix(reqPath, "/img/")
sig, enc, ok := strings.Cut(rest, "/")
if !ok {
return "", false
}
if i := strings.LastIndex(enc, "."); i >= 0 {
enc = enc[:i]
}
raw, err := base64.RawURLEncoding.DecodeString(enc)
if err != nil {
return "", false
}
src := string(raw)
if !hmac.Equal([]byte(sig), []byte(p.sign(src))) {
return "", false
}
if u, err := url.Parse(src); err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" {
return "", false
}
return src, true
}
// ServeHTTP answers GET /img/<signature>/<encoded>.<ext>.
func (p *Proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) {
src, ok := p.parse(r.URL.Path)
if !ok {
http.Error(w, "not a proxied image", http.StatusForbidden)
return
}
base := p.base(src)
m, cached := p.meta(base)
age := p.ageOf(m)
switch {
case cached && age < TTL:
// Fresh: nothing to do but serve it.
case cached && age < TTL+StaleFor:
// Stale: this reader gets it now, the refresh runs behind them.
p.refreshLater(src)
default:
served, started, err := p.fetchAndServe(r.Context(), src, base, w)
if started {
// The response has begun. Whatever happened after that, the
// only honest end is to stop writing: an error page appended
// to half an image corrupts the image and says nothing a
// browser can read.
return
}
if err != nil {
if !cached {
http.Error(w, "image unavailable", http.StatusBadGateway)
return
}
break // an unreachable upstream must not lose a cached image
}
if served {
return
}
}
p.serveCached(w, r, base)
}
// serveCached answers from the cache file, which gives conditional
// requests and ranges for free and streams rather than buffering.
func (p *Proxy) serveCached(w http.ResponseWriter, r *http.Request, base string) {
m, ok := p.meta(base)
if !ok {
http.Error(w, "image unavailable", http.StatusBadGateway)
return
}
f, err := os.Open(base + ".bin")
if err != nil {
http.Error(w, "image unavailable", http.StatusBadGateway)
return
}
defer f.Close()
info, err := f.Stat()
if err != nil {
http.Error(w, "image unavailable", http.StatusBadGateway)
return
}
p.headers(w, m)
// ServeContent answers If-None-Match with 304 on its own.
http.ServeContent(w, r, "", info.ModTime(), f)
}
func (p *Proxy) headers(w http.ResponseWriter, m meta) {
h := w.Header()
h.Set("Content-Type", m.Type)
h.Set("X-Content-Type-Options", "nosniff")
h.Set("Content-Security-Policy", "default-src 'none'; style-src 'unsafe-inline'; sandbox")
// max-age is the browser's, s-maxage the edge's, and
// stale-while-revalidate lets both serve the old copy during a
// refresh instead of making somebody wait.
h.Set("Cache-Control", fmt.Sprintf("public, max-age=%d, s-maxage=%d, stale-while-revalidate=%d",
int(TTL.Seconds()), int(EdgeTTL.Seconds()), int(StaleFor.Seconds())))
if m.ETag != "" {
h.Set("ETag", m.ETag)
}
}
// fetchAndServe streams the upstream response to the reader and into the
// cache at once, so the first byte moves before the last one arrives.
//
// It answers three things: whether it served the request, whether it had
// already written a byte when it stopped, and what went wrong. The middle
// one matters because a response cannot be taken back: an oversize image
// or a connection that dies mid-body leaves a truncated image, and the
// caller must add nothing to it.
//
// A streamed response carries no ETag. The tag is a hash of the bytes,
// which is not known until the last one has gone; every later reader is
// served from the cache and gets one.
func (p *Proxy) fetchAndServe(ctx context.Context, src, base string, w http.ResponseWriter) (served, started bool, err error) {
lock := p.lockFor(base)
lock.Lock()
defer lock.Unlock()
// Another reader may have fetched it while this one waited.
if m, ok := p.meta(base); ok && p.ageOf(m) < TTL {
return false, false, nil
}
resp, ctype, err := p.open(ctx, src)
if err != nil {
return false, false, err
}
defer resp.Body.Close()
tmp, err := os.CreateTemp(p.dir, "fetch-*")
if err != nil {
return false, false, err
}
defer os.Remove(tmp.Name())
defer tmp.Close()
p.headers(w, meta{Type: ctype})
sum := sha256.New()
var buf []byte // the first chunk, kept for the size probe
var n int64
oversize := false
body := io.LimitReader(resp.Body, MaxBytes+1)
chunk := make([]byte, 32<<10)
for {
got, rerr := body.Read(chunk)
if got > 0 {
if n+int64(got) > MaxBytes {
// Past the cap. Nothing more is written, nothing is
// cached, and the reader keeps what already left.
oversize = true
break
}
n += int64(got)
if len(buf) < 64<<10 {
buf = append(buf, chunk[:got]...)
}
sum.Write(chunk[:got])
if _, werr := tmp.Write(chunk[:got]); werr != nil {
return false, started, werr
}
if _, werr := w.Write(chunk[:got]); werr != nil {
// The reader left. The cache write still finishes below.
break
}
started = true
if f, ok := w.(http.Flusher); ok {
f.Flush()
}
}
if rerr == io.EOF {
break
}
if rerr != nil {
return false, started, rerr
}
}
if oversize {
return false, started, fmt.Errorf("imgproxy: larger than %d bytes", MaxBytes)
}
m := meta{Type: ctype, ETag: `"` + hex.EncodeToString(sum.Sum(nil)[:16]) + `"`,
Fetched: p.now().UTC().Format(time.RFC3339)}
m.Width, m.Height = sizeOf(ctype, buf)
p.commit(base, tmp, m)
return true, started, nil
}
// refresh fetches into the cache without a reader attached.
func (p *Proxy) refresh(ctx context.Context, src, base string) error {
lock := p.lockFor(base)
lock.Lock()
defer lock.Unlock()
resp, ctype, err := p.open(ctx, src)
if err != nil {
return err
}
defer resp.Body.Close()
tmp, err := os.CreateTemp(p.dir, "fetch-*")
if err != nil {
return err
}
defer os.Remove(tmp.Name())
defer tmp.Close()
sum := sha256.New()
head := make([]byte, 0, 64<<10)
n, err := io.Copy(io.MultiWriter(tmp, sum, headWriter{&head}), io.LimitReader(resp.Body, MaxBytes+1))
if err != nil {
return err
}
if n > MaxBytes {
return fmt.Errorf("imgproxy: larger than %d bytes", MaxBytes)
}
m := meta{Type: ctype, ETag: `"` + hex.EncodeToString(sum.Sum(nil)[:16]) + `"`,
Fetched: p.now().UTC().Format(time.RFC3339)}
m.Width, m.Height = sizeOf(ctype, head)
p.commit(base, tmp, m)
return nil
}
// headWriter keeps the first bytes of a stream, which is all the size
// probe needs.
type headWriter struct{ into *[]byte }
func (h headWriter) Write(b []byte) (int, error) {
if room := (64 << 10) - len(*h.into); room > 0 {
if len(b) < room {
room = len(b)
}
*h.into = append(*h.into, b[:room]...)
}
return len(b), nil
}
// refreshLater runs one refresh for this URL in the background. The
// reader it was triggered by has already been answered.
func (p *Proxy) refreshLater(src string) {
base := p.base(src)
p.mu.Lock()
if p.refreshing[src] {
p.mu.Unlock()
return
}
p.refreshing[src] = true
p.mu.Unlock()
go func() {
defer func() {
p.mu.Lock()
delete(p.refreshing, src)
p.mu.Unlock()
}()
// Its own deadline: the request that started it is long gone.
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
_ = p.refresh(ctx, src, base)
}()
}
var errNotImage = errors.New("imgproxy: not an image")
// open starts the upstream request and checks everything that can be
// known before the body is read.
func (p *Proxy) open(ctx context.Context, src string) (*http.Response, string, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, src, nil)
if err != nil {
return nil, "", err
}
req.Header.Set("User-Agent", "quarry-imgproxy")
req.Header.Set("Accept", "image/*")
resp, err := p.client.Do(req)
if err != nil {
return nil, "", err
}
if resp.StatusCode != http.StatusOK {
resp.Body.Close()
return nil, "", fmt.Errorf("imgproxy: upstream %d", resp.StatusCode)
}
mt, _, err := mime.ParseMediaType(resp.Header.Get("Content-Type"))
if err != nil || !strings.HasPrefix(mt, "image/") {
resp.Body.Close()
return nil, "", errNotImage
}
if resp.ContentLength > MaxBytes {
resp.Body.Close()
return nil, "", fmt.Errorf("imgproxy: larger than %d bytes", MaxBytes)
}
return resp, mt, nil
}
// commit renames the fetched file and its metadata into place, so a
// reader never sees half an image.
func (p *Proxy) commit(base string, tmp *os.File, m meta) {
if err := tmp.Sync(); err != nil {
return
}
blob, err := json.Marshal(m)
if err != nil {
return
}
if err := os.Rename(tmp.Name(), base+".bin"); err != nil {
return
}
if err := os.WriteFile(base+".meta.tmp", blob, 0o640); err == nil {
os.Rename(base+".meta.tmp", base+".meta")
}
// After a fetch, not on a timer: an idle forge sweeps nothing, and a
// busy one at most once every sweepEvery.
p.maybeSweep()
}
// maybeSweep runs a sweep unless one ran lately.
func (p *Proxy) maybeSweep() {
p.mu.Lock()
if p.now().Sub(p.swept) < sweepEvery {
p.mu.Unlock()
return
}
p.swept = p.now()
p.mu.Unlock()
p.sweep()
}
// sweep drops what the cache cannot serve, and past the budget what
// nobody has asked for in longest.
//
// Nothing here reports an error: every file is a copy of something
// fetchable again, so a removal that fails costs disk and nothing else.
// Removing a file that a reader is part-way through serving is safe —
// the open descriptor outlives the name.
func (p *Proxy) sweep() {
entries, err := os.ReadDir(p.dir)
if err != nil {
return
}
metas := map[string]bool{}
bins := map[string]int64{}
for _, e := range entries {
name := e.Name()
info, err := e.Info()
if err != nil {
continue
}
switch {
case strings.HasPrefix(name, "fetch-"):
// In flight, or left by a fetch that died.
if p.now().Sub(info.ModTime()) > tempAge {
os.Remove(filepath.Join(p.dir, name))
}
case strings.HasSuffix(name, ".type"):
// The first version's metadata. Nothing reads it now, and the
// bytes beside it have no .meta, so they go below.
os.Remove(filepath.Join(p.dir, name))
case strings.HasSuffix(name, ".tmp"):
// A rename that never happened.
if p.now().Sub(info.ModTime()) > tempAge {
os.Remove(filepath.Join(p.dir, name))
}
case strings.HasSuffix(name, ".meta"):
metas[strings.TrimSuffix(name, ".meta")] = true
case strings.HasSuffix(name, ".bin"):
bins[strings.TrimSuffix(name, ".bin")] = info.Size()
}
}
type entry struct {
base string
size int64
fetched time.Time
}
var live []entry
var total int64
for name, size := range bins {
base := filepath.Join(p.dir, name)
m, ok := p.meta(base)
if !metas[name] || !ok {
// Bytes nothing can describe: whatever wrote them, no reader
// can be told what they are.
os.Remove(base + ".bin")
os.Remove(base + ".meta")
continue
}
if p.ageOf(m) > TTL+StaleFor {
// Past the point where even a stale copy may be served.
os.Remove(base + ".bin")
os.Remove(base + ".meta")
continue
}
at, _ := time.Parse(time.RFC3339, m.Fetched)
live = append(live, entry{base: base, size: size, fetched: at})
total += size
}
for name := range metas {
if _, ok := bins[name]; !ok {
os.Remove(filepath.Join(p.dir, name+".meta"))
}
}
if total <= p.budget {
return
}
sort.Slice(live, func(i, j int) bool { return live[i].fetched.Before(live[j].fetched) })
for _, e := range live {
if total <= p.budget {
return
}
os.Remove(e.base + ".bin")
os.Remove(e.base + ".meta")
total -= e.size
}
}
func (p *Proxy) meta(base string) (meta, bool) {
blob, err := os.ReadFile(base + ".meta")
if err != nil {
return meta{}, false
}
var m meta
if err := json.Unmarshal(blob, &m); err != nil {
return meta{}, false
}
if _, err := os.Stat(base + ".bin"); err != nil {
return meta{}, false
}
return m, true
}
func (p *Proxy) ageOf(m meta) time.Duration {
at, err := time.Parse(time.RFC3339, m.Fetched)
if err != nil {
return TTL + StaleFor + time.Hour // unreadable: treat as long expired
}
return p.now().Sub(at)
}
func (p *Proxy) lockFor(base string) *sync.Mutex {
p.mu.Lock()
defer p.mu.Unlock()
l, ok := p.locks[base]
if !ok {
l = &sync.Mutex{}
p.locks[base] = l
}
return l
}
// svgSize reads width and height off an SVG root, or its viewBox, which
// is what badge generators emit.
var (
svgTag = regexp.MustCompile(`(?is)<svg\b([^>]*)>`)
svgAttr = regexp.MustCompile(`(?is)\b(width|height)\s*=\s*"([0-9.]+)(?:px)?"`)
svgViewBox = regexp.MustCompile(`(?is)\bviewBox\s*=\s*"\s*[-0-9.]+\s+[-0-9.]+\s+([0-9.]+)\s+([0-9.]+)`)
)
// sizeOf is the image's pixel size, or zeros when it cannot be read from
// the bytes in hand. Never an error: a size is an optimisation, and a
// missing one only costs the reflow this saves.
func sizeOf(ctype string, head []byte) (int, int) {
if strings.HasPrefix(ctype, "image/svg") {
tag := svgTag.FindSubmatch(head)
if tag == nil {
return 0, 0
}
attrs := tag[1]
var w, h int
for _, m := range svgAttr.FindAllSubmatch(attrs, -1) {
v, _ := strconv.ParseFloat(string(m[2]), 64)
if strings.EqualFold(string(m[1]), "width") {
w = int(v)
} else {
h = int(v)
}
}
if w > 0 && h > 0 {
return w, h
}
if m := svgViewBox.FindSubmatch(attrs); m != nil {
vw, _ := strconv.ParseFloat(string(m[1]), 64)
vh, _ := strconv.ParseFloat(string(m[2]), 64)
return int(vw), int(vh)
}
return 0, 0
}
cfg, _, err := image.DecodeConfig(strings.NewReader(string(head)))
if err != nil {
return 0, 0
}
return cfg.Width, cfg.Height
}
// lookupIP is the resolver the guard uses, named so a test can substitute
// one.
var lookupIP = func(ctx context.Context, host string) ([]net.IPAddr, error) {
return net.DefaultResolver.LookupIPAddr(ctx, host)
}
// reserved is an address a server-side fetcher has no business reaching
// on a document author's say-so.
func reserved(ip net.IP) bool {
return ip.IsLoopback() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() ||
ip.IsPrivate() || ip.IsUnspecified() || ip.IsMulticast() ||
ip.IsInterfaceLocalMulticast()
}
// guardedClient dials only the address it checked, so a name cannot
// resolve to one address for the check and another for the connection.
// Redirects are followed a few times; every hop dials through the guard.
func guardedClient(allowPrivate bool) *http.Client {
tr := http.DefaultTransport.(*http.Transport).Clone()
d := &net.Dialer{Timeout: 5 * time.Second}
tr.DialContext = func(ctx context.Context, network, addr string) (net.Conn, error) {
host, port, err := net.SplitHostPort(addr)
if err != nil {
return nil, err
}
ips, err := lookupIP(ctx, host)
if err != nil {
return nil, err
}
if len(ips) == 0 {
return nil, fmt.Errorf("imgproxy: no address for %s", host)
}
if !allowPrivate {
for _, ip := range ips {
if reserved(ip.IP) {
return nil, fmt.Errorf("imgproxy: refusing a private address (%s)", ip.IP)
}
}
}
return d.DialContext(ctx, network, net.JoinHostPort(ips[0].IP.String(), port))
}
return &http.Client{
Timeout: 10 * time.Second,
Transport: tr,
CheckRedirect: func(req *http.Request, via []*http.Request) error {
if len(via) >= 3 {
return errors.New("imgproxy: too many redirects")
}
if req.URL.Scheme != "http" && req.URL.Scheme != "https" {
return errors.New("imgproxy: redirect to a non-http scheme")
}
return nil
},
}
}