// foreign.go checks a promise rather than a service: that the page a
// reader receives loads nothing from another host.
//
// quarry's privacy page states it plainly — no external stylesheet,
// font, image or script anywhere in the interface — and the whole
// interface is built to keep it: identicons are drawn locally, the
// diagrams are rendered server-side, the one script bundle is served
// from this origin under a nonce.
//
// What the forge sends is not what the reader receives, though. Between
// them sits whatever the operator put in front: a CDN, a proxy, a
// company middlebox. On zeitforge, Cloudflare's Web Analytics feature
// appended a beacon script to every page, and the reader's browser
// resolved, connected and completed a TLS handshake with a third party
// before the page's own Content-Security-Policy stopped the script from
// running. The forge served a page that kept its promise; the reader
// got one that did not, and nothing on either side noticed.
//
// So the check reads the page from OUTSIDE — over the public address,
// through whatever the origin sits behind — and asks the question the
// promise makes: does anything here load from another host?
package health
import (
"context"
"fmt"
"io"
"net/http"
"net/url"
"sort"
"strings"
"sync"
"time"
"golang.org/x/net/html"
)
// maxPage bounds what the probe will read. The login page is ~26KB; a
// megabyte is far past anything this forge serves and stops a
// misconfigured intermediary from feeding the probe a stream.
const maxPage = 1 << 20
// loaders are the element/attribute pairs that make the browser FETCH
// something. Deliberately not <a href>: a link is an address a reader
// may choose to follow, not a resource the page pulls in — quarry's own
// footer links to fossil-scm.org, and calling that a violation would
// make the check cry wolf on every page it ever ran against.
var loaders = map[string]string{
"script": "src",
"link": "href",
"img": "src",
"iframe": "src",
"frame": "src",
"embed": "src",
"source": "src",
"track": "src",
"audio": "src",
"video": "src",
"input": "src",
"object": "data",
}
// ForeignResources returns the foreign hosts a page pulls from, sorted
// and deduplicated. Empty means the promise held.
//
// Relative URLs, fragments and same-host absolutes are all ours. A
// data: or blob: URL is ours too — only the page's own markup can
// produce one, and it reaches no network.
func ForeignResources(pageURL string, body io.Reader) ([]string, error) {
base, err := url.Parse(pageURL)
if err != nil {
return nil, err
}
doc, err := html.Parse(io.LimitReader(body, maxPage))
if err != nil {
return nil, err
}
found := map[string]bool{}
var walk func(*html.Node)
walk = func(n *html.Node) {
if n.Type == html.ElementNode {
if attr, ok := loaders[n.Data]; ok {
for _, a := range n.Attr {
if a.Key != attr {
continue
}
if host := foreignHost(base, a.Val); host != "" {
found[host] = true
}
}
}
// srcset carries several candidates, each "URL [descriptor]".
for _, a := range n.Attr {
if a.Key != "srcset" {
continue
}
for _, cand := range strings.Split(a.Val, ",") {
fields := strings.Fields(cand)
if len(fields) == 0 {
continue
}
if host := foreignHost(base, fields[0]); host != "" {
found[host] = true
}
}
}
}
for c := n.FirstChild; c != nil; c = c.NextSibling {
walk(c)
}
}
walk(doc)
out := make([]string, 0, len(found))
for h := range found {
out = append(out, h)
}
sort.Strings(out)
return out, nil
}
// foreignHost returns the host a reference points at when that host is
// not the page's own, else "".
func foreignHost(base *url.URL, ref string) string {
ref = strings.TrimSpace(ref)
if ref == "" {
return ""
}
u, err := url.Parse(ref)
if err != nil {
// An unparseable reference fetches nothing, so it cannot be a
// foreign fetch. Reporting it would be noise about a typo.
return ""
}
// data: and blob: carry their payload; they reach no network.
if u.Scheme == "data" || u.Scheme == "blob" {
return ""
}
abs := base.ResolveReference(u)
if abs.Host == "" || strings.EqualFold(abs.Host, base.Host) {
return ""
}
// Lowercased, because a host name is case-insensitive and the
// caller deduplicates by this string: CDN.example and cdn.example
// are one host, and reporting both would name the same violation
// twice.
return strings.ToLower(abs.Host)
}
// ForeignProbe builds the Dep probe that reads the forge's own public
// page and reports any host it pulls from.
//
// It reads over the PUBLIC address on purpose. Probing the origin
// directly would ask the wrong question — the origin is the half that
// was never in doubt — and would pass while readers were being served
// something else entirely.
//
// A probe that cannot reach the page reports that, and does not report
// "clean". The two are not the same, and treating an unanswered
// question as a good answer is how a check becomes decoration. This one
// has a specific way of failing: the front may refuse the probe itself.
// Cloudflare answered this forge's own scripted requests with a 1010
// browser-signature block until they carried an ordinary User-Agent.
// every bounds how often the page is actually fetched. The watcher's
// beat suits a service that can die between two ticks; an injected
// script is a CONFIGURATION, and configurations do not come and go in
// thirty seconds. Between fetches the last verdict stands, so the
// watcher still sees a steady state and still reports only transitions
// — it just stops pulling the forge's own page through its own front
// two thousand times a day to learn nothing.
func ForeignProbe(client *http.Client, every time.Duration) func(context.Context, Dep) error {
if client == nil {
client = &http.Client{Timeout: 10 * time.Second}
}
if every <= 0 {
every = 15 * time.Minute
}
var (
mu sync.Mutex
last time.Time
result error
known bool
)
return func(ctx context.Context, d Dep) error {
mu.Lock()
if known && time.Since(last) < every {
held := result
mu.Unlock()
return held
}
mu.Unlock()
err := readPublicPage(ctx, client, d)
mu.Lock()
last, result, known = time.Now(), err, true
mu.Unlock()
return err
}
}
// readPublicPage does one read and one verdict.
func readPublicPage(ctx context.Context, client *http.Client, d Dep) error {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, d.URL, nil)
if err != nil {
return err
}
// An ordinary browser's User-Agent, because a front that
// filters robots would otherwise answer the probe differently
// from how it answers a reader — and how it answers a reader is
// the entire question.
req.Header.Set("User-Agent", probeAgent)
resp, err := client.Do(req)
if err != nil {
return fmt.Errorf("could not read the public page: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode >= 400 {
return fmt.Errorf("the public page answered %s, so this check could not run", resp.Status)
}
hosts, err := ForeignResources(d.URL, resp.Body)
if err != nil {
return fmt.Errorf("could not read the public page: %w", err)
}
if len(hosts) == 0 {
return nil
}
return fmt.Errorf("readers are being served resources from %s, which this forge did not send",
strings.Join(hosts, ", "))
}
// probeAgent is what the check presents itself as. Honest about being
// quarry, ordinary enough not to be filtered.
const probeAgent = "Mozilla/5.0 (compatible; quarry-origin-check)"