// Package errlog is the server's error journal: a bounded, deduplicated
// ring of what has gone wrong, so 500s stop being writes to a log
// nobody reads. Every funnel that answers "internal error" to a reader
// reports the real error here; /fleet renders the ring; the notify
// plane fires on a signature's FIRST sighting only.
//
// Deliberately in-process state, not a storage tier: the ring restarts
// with the server, which is the right lifetime for "what is going wrong
// right now". Durability, when someone needs it, is fossil's answer —
// an append-only errorlog file — layered beside this, not instead.
package errlog
import (
"fmt"
"io"
"sync"
"time"
)
// Entry is one distinct error signature and its history.
type Entry struct {
Message string `json:"message"`
Surface string `json:"surface"` // web | api | panic | store | ingest | ci | mirror
Path string `json:"path"` // where it was FIRST seen
Count int `json:"count"`
First time.Time `json:"first"`
Last time.Time `json:"last"`
// Repo attributes the failure when one is in scope. A fleet serving
// two dozen stores answers "what is broken" uselessly if it cannot
// also answer "where" — and the same message from two repositories
// is usually two faults, not one.
Repo string `json:"repo,omitempty"`
// Trace is the stack at the moment of the panic, kept for the FIRST
// sighting only. A repeat adds nothing but bytes: the signature is
// the same code path, and the first stack already describes it.
Trace string `json:"trace,omitempty"`
}
// Ring holds the most recent distinct signatures, deduplicated by
// message: a hot loop is one row with a count, never a flood.
type Ring struct {
mu sync.Mutex
cap int
entries map[string]*Entry
// OnFirst, when set, runs on a signature's first sighting — the
// alerting hook. Called OUTSIDE the lock with a copy; an alert
// channel that repeats itself gets muted, so repeats never fire it.
OnFirst func(Entry)
// out, when set, receives one plain line per occurrence — the ring
// restarts with the server, the file is what survives for
// post-restart forensics (fossil's --errorlog, same idea).
out io.Writer
// outMu serialises writes to out, separately from mu. Reporting an
// error must not wait on somebody else's disk.
outMu sync.Mutex
}
// SetOutput directs a plain line per occurrence to w (an append-opened
// file in the real wiring). Best-effort: a full disk must not turn
// error RECORDING into error THROWING.
func (r *Ring) SetOutput(w io.Writer) {
r.mu.Lock()
r.out = w
r.mu.Unlock()
}
// journal appends one plain line per occurrence, serialised on its own
// mutex. Best-effort by contract: a failure to write must never become
// a failure to record.
func (r *Ring) journal(now time.Time, surface, path, msg string) {
r.mu.Lock()
w := r.out
r.mu.Unlock()
if w == nil {
return
}
r.outMu.Lock()
fmt.Fprintf(w, "%s %s %s: %s\n", now.UTC().Format(time.RFC3339), surface, path, msg)
r.outMu.Unlock()
}
// New builds a ring bounded to cap distinct signatures.
func New(cap int) *Ring {
if cap <= 0 {
cap = 500
}
return &Ring{cap: cap, entries: make(map[string]*Entry)}
}
// Default is the process-wide journal the funnels report into.
var Default = New(500)
// Report records one occurrence. First sighting of a signature fires
// OnFirst; when the ring is full, the signature least recently SEEN is
// evicted — an error still happening is never displaced by one that
// stopped.
func (r *Ring) Report(surface, path string, err error) {
r.ReportIn(surface, path, "", "", err)
}
// ReportIn is Report with the repository the failure belongs to and,
// for a panic, the stack at the moment it happened. Split rather than
// widened so the three original call sites read as they did.
func (r *Ring) ReportIn(surface, path, repo, trace string, err error) {
if err == nil {
return
}
msg := err.Error()
now := time.Now()
// The journal write happens OFF the ring's lock, on a lock of its
// own. It is an unbuffered append to a file — deliberately, since
// this log exists for forensics after a crash and a buffer is
// exactly what a crash loses — and holding the ring's mutex across
// a write(2) meant every panic, every failed page and every
// browser report queued behind one disk. They contend for the file
// still; they no longer contend for the ring.
r.journal(now, surface, path, msg)
r.mu.Lock()
e, seen := r.entries[msg]
if seen {
e.Count++
e.Last = now
r.mu.Unlock()
return
}
if len(r.entries) >= r.cap {
var oldestKey string
var oldest time.Time
for k, v := range r.entries {
if oldestKey == "" || v.Last.Before(oldest) {
oldestKey, oldest = k, v.Last
}
}
delete(r.entries, oldestKey)
}
fresh := &Entry{Message: msg, Surface: surface, Path: path,
Count: 1, First: now, Last: now, Repo: repo, Trace: trace}
r.entries[msg] = fresh
hook := r.OnFirst
copyE := *fresh
r.mu.Unlock()
if hook != nil {
hook(copyE)
}
}
// Recent returns entries newest-last-seen first, at most limit.
func (r *Ring) Recent(limit int) []Entry {
r.mu.Lock()
out := make([]Entry, 0, len(r.entries))
for _, e := range r.entries {
out = append(out, *e)
}
r.mu.Unlock()
for i := 1; i < len(out); i++ {
for j := i; j > 0 && out[j].Last.After(out[j-1].Last); j-- {
out[j], out[j-1] = out[j-1], out[j]
}
}
if limit > 0 && len(out) > limit {
out = out[:limit]
}
return out
}
// Stats answers the health surface: total occurrences, distinct
// signatures, and when anything last went wrong.
func (r *Ring) Stats() (total, distinct int, lastAt time.Time) {
r.mu.Lock()
defer r.mu.Unlock()
for _, e := range r.entries {
total += e.Count
distinct++
if e.Last.After(lastAt) {
lastAt = e.Last
}
}
return total, distinct, lastAt
}