// exception.go persists the error journal. The in-memory ring
// (internal/errlog) is fast, bounded and deliberately lossy; it also
// dies with the process, which meant the Exceptions panel could only
// ever answer "what has broken since the last restart" — and a restart
// is frequently the thing that happened *because* something broke.
//
// This is the other half: one row per distinct signature, counted and
// dated, surviving restarts. Social state, not truth (RFC 0001 §11) —
// losing it loses a diagnosis, never a fact the DAG holds.
package oltp
import (
"database/sql"
"encoding/json"
"sync/atomic"
"time"
"craftdesign.group/quarry/internal/fault"
)
const exceptionSchema = `
CREATE TABLE IF NOT EXISTS exception (
sig TEXT NOT NULL PRIMARY KEY, -- stable hash of the message
message TEXT NOT NULL,
surface TEXT NOT NULL, -- web | api | panic | browser | store | ingest | ci | mirror | chat | template
path TEXT NOT NULL, -- where it was FIRST seen
repo TEXT NOT NULL DEFAULT '',
trace TEXT NOT NULL DEFAULT '',
count INTEGER NOT NULL,
first TEXT NOT NULL,
last TEXT NOT NULL,
-- Resolved marks a signature the maintainer has dealt with. It
-- REOPENS on the next occurrence: an error that came back is not
-- resolved, whatever anybody clicked.
resolved_at TEXT NOT NULL DEFAULT ''
);
-- The panel orders by what is still happening, so that is the index.
CREATE INDEX IF NOT EXISTS exception_last ON exception(last DESC);
-- One row per occurrence, kept briefly. The counted row above cannot
-- tell a fire from a finished incident: 833 occurrences over two days
-- and 150 inside one 38-minute window are the same number and opposite
-- situations. A rate needs WHEN, and when needs a row each.
--
-- Deliberately small and deliberately lossy: pruned to a week, because
-- this answers "is it happening NOW" and history belongs to the count.
CREATE TABLE IF NOT EXISTS exception_hit (
sig TEXT NOT NULL,
at TEXT NOT NULL
);
CREATE INDEX IF NOT EXISTS exception_hit_sig ON exception_hit(sig, at);`
// exceptionMigrations widen older stores with what a reader needs to
// act: the raiser's account of the cost and the fix, the specifics as
// data, the build that last saw it, and the request it belonged to.
var exceptionMigrations = []string{
`ALTER TABLE exception ADD COLUMN why TEXT NOT NULL DEFAULT ''`,
`ALTER TABLE exception ADD COLUMN fix TEXT NOT NULL DEFAULT ''`,
`ALTER TABLE exception ADD COLUMN fields TEXT NOT NULL DEFAULT ''`,
`ALTER TABLE exception ADD COLUMN build TEXT NOT NULL DEFAULT ''`,
`ALTER TABLE exception ADD COLUMN req TEXT NOT NULL DEFAULT ''`,
}
// Exception is one persisted signature and its history.
type Exception struct {
Sig string
Message string
Surface string
Path string
Repo string
Trace string
Count int
First time.Time
Last time.Time
Resolved bool
// Why the failure matters, Fix what to do, Fields the specifics —
// all from the fault the raiser built (internal/fault). Refreshed
// on each occurrence, because they describe the current state.
Why string
Fix string
Fields []fault.Field
// Build last saw it; Req is the request it belonged to.
Build string
Req string
}
// RecordException upserts one occurrence. The first sighting keeps its
// path and trace; later ones only advance the count and the clock,
// because the first stack already describes the code path and a repeat
// adds nothing but bytes.
//
// A signature that recurs is un-resolved by the same statement that
// counts it: there is no window in which a returning error is both
// happening and marked handled.
func (d *DB) RecordException(e Exception) error {
if e.Sig == "" {
return nil
}
fields, _ := json.Marshal(e.Fields)
d.recordHit(e.Sig, e.Last)
_, err := d.db.Exec(`
INSERT INTO exception (sig, message, surface, path, repo, trace, count, first, last,
why, fix, fields, build, req)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(sig) DO UPDATE SET
count = exception.count + 1,
last = excluded.last,
resolved_at = '',
-- The message and its context describe the LATEST
-- occurrence: a queue depth from an hour ago is not the
-- number anyone is about to act on. The trace is not
-- refreshed — it names the code path, which has not moved.
message = excluded.message,
why = excluded.why,
fix = excluded.fix,
fields = excluded.fields,
build = excluded.build,
req = excluded.req`,
e.Sig, e.Message, e.Surface, e.Path, e.Repo, e.Trace, e.Count,
e.First.UTC().Format(time.RFC3339Nano), e.Last.UTC().Format(time.RFC3339Nano),
e.Why, e.Fix, string(fields), e.Build, e.Req)
return err
}
// Exceptions returns the most recently active signatures, newest first.
func (d *DB) Exceptions(limit int) ([]Exception, error) {
if limit <= 0 {
limit = 200
}
rows, err := d.db.Query(`
SELECT sig, message, surface, path, repo, trace, count, first, last, resolved_at,
why, fix, fields, build, req
FROM exception ORDER BY last DESC LIMIT ?`, limit)
if err != nil {
return nil, err
}
defer rows.Close()
var out []Exception
for rows.Next() {
var e Exception
var first, last, resolved, fields string
if err := rows.Scan(&e.Sig, &e.Message, &e.Surface, &e.Path, &e.Repo,
&e.Trace, &e.Count, &first, &last, &resolved,
&e.Why, &e.Fix, &fields, &e.Build, &e.Req); err != nil {
return nil, err
}
if fields != "" {
_ = json.Unmarshal([]byte(fields), &e.Fields)
}
e.First, _ = time.Parse(time.RFC3339Nano, first)
e.Last, _ = time.Parse(time.RFC3339Nano, last)
e.Resolved = resolved != ""
out = append(out, e)
}
return out, rows.Err()
}
// ExceptionTotals is what the header counts: distinct signatures and
// total occurrences over the whole history, not this page's slice.
func (d *DB) ExceptionTotals() (signatures, occurrences int, err error) {
var occ sql.NullInt64
err = d.db.QueryRow(`SELECT count(*), sum(count) FROM exception`).Scan(&signatures, &occ)
return signatures, int(occ.Int64), err
}
// hitWindow is how much of the past the rate is read over. A week is
// long enough to show a daily rhythm and short enough that the table
// stays small on a busy forge.
const hitWindow = 7 * 24 * time.Hour
// pruneEvery bounds the delete: pruning on every occurrence would run a
// scan per error during exactly the storm the journal exists for.
const pruneEvery = 500
var hitsSincePrune atomic.Int64
// recordHit appends one occurrence and, occasionally, drops what has
// aged out. Best-effort by contract, like the journal itself: failing
// to record a rate must not fail the recording of the error.
func (d *DB) recordHit(sig string, at time.Time) {
if sig == "" {
return
}
_, _ = d.db.Exec(`INSERT INTO exception_hit (sig, at) VALUES (?, ?)`,
sig, at.UTC().Format(time.RFC3339Nano))
if hitsSincePrune.Add(1) < pruneEvery {
return
}
hitsSincePrune.Store(0)
_, _ = d.db.Exec(`DELETE FROM exception_hit WHERE at < ?`,
time.Now().Add(-hitWindow).UTC().Format(time.RFC3339Nano))
}
// Rates reads, for each signature, how often it has happened lately —
// the number that separates "still on fire" from "over".
//
// Returned as counts per bucket, oldest first, so a caller can draw the
// shape as well as state the rate. A signature with no recent hit is
// absent rather than a row of zeroes: it is not happening.
func (d *DB) Rates(buckets int, span time.Duration) (map[string][]int, error) {
if buckets <= 0 {
buckets = 24
}
if span <= 0 {
span = 24 * time.Hour
}
start := time.Now().Add(-span)
rows, err := d.db.Query(`SELECT sig, at FROM exception_hit WHERE at >= ?`,
start.UTC().Format(time.RFC3339Nano))
if err != nil {
return nil, err
}
defer rows.Close()
out := map[string][]int{}
width := span / time.Duration(buckets)
for rows.Next() {
var sig, at string
if err := rows.Scan(&sig, &at); err != nil {
return nil, err
}
t, perr := time.Parse(time.RFC3339Nano, at)
if perr != nil {
continue
}
i := int(t.Sub(start) / width)
switch {
case i < 0:
i = 0
case i >= buckets:
i = buckets - 1
}
if out[sig] == nil {
out[sig] = make([]int, buckets)
}
out[sig][i]++
}
return out, rows.Err()
}
// ResolveException marks a signature handled. It is a note, not a
// promise: the next occurrence clears it (see RecordException).
func (d *DB) ResolveException(sig string) error {
_, err := d.db.Exec(`UPDATE exception SET resolved_at = ? WHERE sig = ?`,
time.Now().UTC().Format(time.RFC3339Nano), sig)
return err
}