package obs
import (
"context"
"hash/maphash"
"net/http"
"net/url"
"strings"
"sync"
"time"
)
// Hit is one request the forge answered, whatever door it came in by.
//
// Render is the ops page's record and covers only what the web and API
// handlers serve. A clone never reaches either: sync is mounted outside
// them, so the one kind of reading that leaves no other trace left none
// here either. Hit is filed by the outermost handler, so every request
// is one row — pages, API calls, sync rounds, runner downloads, assets.
//
// It holds facts, never conclusions. Whether a request came from a
// person, a crawler or a script is decided when the traffic page asks
// (see Classify), so a better rule reclassifies the history it is shown
// rather than only the rows written after it.
type Hit struct {
At time.Time
Door string // DoorWeb, DoorAPI, DoorSync, DoorAsset, DoorRunner, DoorBridge
Method string
URL string // the path, never the query
Repo string
Status int
Bytes int64
Elapsed time.Duration
// Stream is a response that stayed open — a live page, chat, the
// timeline. Its Elapsed is how long a tab stayed open, so it is
// counted as a request and never as a response time.
Stream bool
// Addr is the reader's address. The store keeps it for a day and
// then blanks it; what outlives it is the network and the country.
Addr string
Country string // the edge's two-letter guess, "" when nothing in front says
// ASN and Network are the autonomous system the address belongs to,
// resolved when the hit is filed because the address does not last.
ASN uint32
Network string
// Visitor counts unique readers without remembering any: a hash of
// the address and user agent under a salt that is drawn at random
// each day and never stored. Two requests from one reader on one
// day share it; nothing can turn it back into an address, and
// tomorrow's salt makes today's unlinkable.
Visitor uint64
UA string
// Referrer is the referring site's host, never its path: where a
// reader came from is the useful fact, and the page they were on is
// theirs.
Referrer string
// Account is the identity the forge verified, "" when it verified
// none. Claimed is a name the request offered and nothing checked —
// a fossil login card whose signature did not verify. Kept apart for
// the reason the sync tallies keep them apart: a claim is free to
// forge.
Account string
Claimed string
// Verb is the sync round's kind: clone, fetch or push. "" off sync.
Verb string
// Client and Version are the sync client as the round named it.
Client, Version string
}
// The doors a request can come in by. A closed set: the page groups by
// it.
const (
DoorWeb = "web"
DoorAPI = "api"
DoorSync = "sync"
DoorAsset = "asset"
DoorRunner = "runner"
DoorBridge = "bridge"
)
// HitNote is what the handlers inside know about a request that the
// outermost one cannot: who it was, which repository, what a sync round
// did. The outermost handler puts one on the context and files it with
// the hit once the response is written.
//
// Every method is safe on nil, so a handler notes without asking
// whether anything is listening.
type HitNote struct {
mu sync.Mutex
repo, account, claimed string
verb, client, version string
door string
repoSet bool
}
type ctxHit struct{}
// NoteOf is the request's note, nil when no outermost handler made one.
func NoteOf(ctx context.Context) *HitNote {
if ctx == nil {
return nil
}
n, _ := ctx.Value(ctxHit{}).(*HitNote)
return n
}
// Account records the verified identity. An empty name changes nothing,
// so a handler can note whatever it resolved.
func (n *HitNote) Account(name string) {
if n == nil || name == "" {
return
}
n.mu.Lock()
n.account = name
n.mu.Unlock()
}
// Repo records the repository the request was about, as the handler
// resolved it — an owner's namespaced path names it in the second
// segment, which only the handler knows.
func (n *HitNote) Repo(name string) {
if n == nil || name == "" {
return
}
n.mu.Lock()
n.repo, n.repoSet = name, true
n.mu.Unlock()
}
// Door overrides the door read off the path.
func (n *HitNote) Door(door string) {
if n == nil || door == "" {
return
}
n.mu.Lock()
n.door = door
n.mu.Unlock()
}
// Sync records what a sync round was and who it said it was.
func (n *HitNote) Sync(verb, account, claimed, client, version string) {
if n == nil {
return
}
n.mu.Lock()
n.door, n.verb = DoorSync, verb
if account != "" {
n.account = account
}
n.claimed, n.client, n.version = claimed, client, version
n.mu.Unlock()
}
// Hits files a Hit for every request next answers. addr is the reader's
// address under the forge's trusted-proxy rule, so the address here and
// the one the limiters key on are the same answer.
func (r *Recorder) Hits(next http.Handler, addr func(*http.Request) string) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
start := time.Now()
note := &HitNote{}
cw := &countingWriter{ResponseWriter: w, status: http.StatusOK}
next.ServeHTTP(cw, req.WithContext(context.WithValue(req.Context(), ctxHit{}, note)))
h := Hit{
At: start, Method: req.Method, URL: req.URL.Path,
Status: cw.status, Bytes: cw.bytes, Elapsed: time.Since(start),
Stream: strings.HasPrefix(cw.Header().Get("Content-Type"), "text/event-stream"),
Addr: addr(req),
UA: req.UserAgent(),
Referrer: referrerHost(req.Referer(), req.Host),
}
// The edge's country is believed only where its address is: a
// header a client could write is no evidence of anything.
if h.Addr != remoteHost(req.RemoteAddr) {
h.Country = strings.ToUpper(strings.TrimSpace(req.Header.Get("CF-IPCountry")))
if h.Country == "XX" || h.Country == "T1" {
h.Country = ""
}
}
note.mu.Lock()
h.Door, h.Repo = note.door, note.repo
h.Account, h.Claimed = note.account, note.claimed
h.Verb, h.Client, h.Version = note.verb, note.client, note.version
repoSet := note.repoSet
note.mu.Unlock()
if h.Door == "" {
h.Door = doorOf(req)
}
if !repoSet {
h.Repo = repoOf(req.URL.Path)
}
r.RecordHit(h)
})
}
// NetworkOf names the network an address belongs to.
type NetworkOf func(addr string) (asn uint32, name string, ok bool)
// SetNetworks is how the recorder learns which network an address is
// in. Without it hits are filed with none.
func (r *Recorder) SetNetworks(f NetworkOf) {
r.mu.Lock()
r.networks = f
r.mu.Unlock()
}
// RecordHit files one request with the sink. There is no ring: nothing
// live reads hits, and a forge with history switched off has no traffic
// page to show.
func (r *Recorder) RecordHit(h Hit) {
if r == nil {
return
}
r.mu.Lock()
sink, netOf := r.sink, r.networks
r.mu.Unlock()
if h.Addr != "" && h.Visitor == 0 {
h.Visitor = r.visitor(h.At, h.Addr, h.UA)
}
if netOf != nil && h.ASN == 0 && h.Addr != "" {
if asn, name, ok := netOf(h.Addr); ok {
h.ASN, h.Network = asn, NetworkName(asn, name)
}
}
if s, ok := sink.(interface{ Hit(Hit) }); ok {
s.Hit(h)
}
}
// doorOf reads the door off the path, for requests no handler named.
func doorOf(req *http.Request) string {
p := req.URL.Path
switch {
case strings.HasPrefix(p, "/api/"):
return DoorAPI
case strings.HasPrefix(p, "/assets/"):
return DoorAsset
case strings.HasPrefix(p, "/runner/"):
return DoorRunner
case strings.Contains(p, "/ci/bridge/"):
return DoorBridge
}
return DoorWeb
}
// repoOf is the first segment after /r/, the flat form. A namespaced
// path is the handler's to resolve, through HitNote.Repo.
func repoOf(p string) string {
rest, ok := strings.CutPrefix(p, "/r/")
if !ok {
return ""
}
name, _, _ := strings.Cut(rest, "/")
if u, err := url.PathUnescape(name); err == nil {
return u
}
return name
}
// referrerHost is the referring site, "" for none and for the forge
// itself: a reader moving between its own pages came from nowhere new.
func referrerHost(ref, self string) string {
if ref == "" {
return ""
}
u, err := url.Parse(ref)
if err != nil || u.Host == "" {
return ""
}
host := strings.ToLower(u.Hostname())
if strings.EqualFold(host, hostOnly(self)) {
return ""
}
return host
}
func hostOnly(h string) string {
if i := strings.LastIndexByte(h, ':'); i >= 0 && !strings.Contains(h[i:], "]") {
return h[:i]
}
return h
}
func remoteHost(a string) string {
if i := strings.LastIndexByte(a, ':'); i >= 0 {
return strings.Trim(a[:i], "[]")
}
return a
}
// countingWriter remembers the status and counts the body bytes.
type countingWriter struct {
http.ResponseWriter
status int
bytes int64
written bool
}
func (w *countingWriter) WriteHeader(code int) {
if !w.written {
w.status, w.written = code, true
}
w.ResponseWriter.WriteHeader(code)
}
func (w *countingWriter) Write(b []byte) (int, error) {
w.written = true
n, err := w.ResponseWriter.Write(b)
w.bytes += int64(n)
return n, err
}
// Flush passes through, so a stream behind this writer still streams.
func (w *countingWriter) Flush() {
if f, ok := w.ResponseWriter.(http.Flusher); ok {
f.Flush()
}
}
// Unwrap lets http.ResponseController reach the writer underneath —
// deadlines, hijacking — through this one.
func (w *countingWriter) Unwrap() http.ResponseWriter { return w.ResponseWriter }
// visitor is the day's hash of a reader. The seed is per day and lives
// only in memory, so a restart starts a new one: a reader seen before
// and after it counts twice that day, which is the price of a salt that
// is never written down.
func (r *Recorder) visitor(at time.Time, addr, ua string) uint64 {
day := at.UTC().Format("2006-01-02")
r.mu.Lock()
if r.saltDay != day {
r.salt, r.saltDay = maphash.MakeSeed(), day
}
seed := r.salt
r.mu.Unlock()
var h maphash.Hash
h.SetSeed(seed)
h.WriteString(addr)
h.WriteByte(0)
h.WriteString(ua)
// 63 bits: database/sql refuses an unsigned value with its high bit
// set, and one such key failed every batch it was in — renders too.
return h.Sum64() >> 1
}