package api
import (
"context"
"errors"
"sort"
"sync"
"testing"
"time"
"craftdesign.group/quarry/internal/obs"
"craftdesign.group/quarry/internal/oltp"
"craftdesign.group/quarry/internal/ops"
)
type memPrefs struct {
mu sync.Mutex
m map[string]string
}
func (p *memPrefs) AccountPref(a, k string) string {
p.mu.Lock()
defer p.mu.Unlock()
return p.m[a+"|"+k]
}
func (p *memPrefs) SetAccountPref(a, k, v string) error {
p.mu.Lock()
defer p.mu.Unlock()
if p.m == nil {
p.m = map[string]string{}
}
if v == "" {
delete(p.m, a+"|"+k)
} else {
p.m[a+"|"+k] = v
}
return nil
}
type alertRig struct {
fn *fakeNotify
prefs *memPrefs
a *OpsAlerter
now time.Time
}
func newAlertRig(people ...string) *alertRig {
r := &alertRig{fn: &fakeNotify{}, prefs: &memPrefs{}, now: time.Date(2026, 11, 15, 12, 0, 0, 0, time.UTC)}
r.a = &OpsAlerter{
Deps: &Deps{Notify: r.fn, Origin: "https://forge.example"},
People: func() []string { return people },
State: r.prefs,
Now: func() time.Time { return r.now },
}
return r
}
// sent waits for the asynchronous hand-offs to settle and answers what
// was triggered, as workflow:to:phase.
func (r *alertRig) sent() []string {
time.Sleep(150 * time.Millisecond)
var out []string
for _, e := range r.fn.snapshot() {
p, _ := e.Payload["phase"].(string)
out = append(out, e.Workflow+":"+e.To+":"+p)
}
sort.Strings(out)
return out
}
func warn(kind string) obs.MemWarning {
return obs.MemWarning{Kind: kind, Value: 123, Threshold: 456}
}
func TestOpsAlertStartAndClearEachSendOnce(t *testing.T) {
r := newAlertRig("dana", "eli")
r.a.WarningStarted(warn(obs.WarnKindHost), r.now)
if got := r.sent(); len(got) != 2 || got[0] != "quarry-ops-alert:dana:start" || got[1] != "quarry-ops-alert:eli:start" {
t.Fatalf("FAIL: start sends once to each person: %v", got)
}
r.now = r.now.Add(10 * time.Minute)
r.a.WarningCleared(warn(obs.WarnKindHost), obs.MemSample{At: r.now}, 10*time.Minute)
if got := r.sent(); len(got) != 4 {
t.Errorf("FAIL: clear sends once to each person: %v", got)
}
r.a.WarningCleared(warn(obs.WarnKindHost), obs.MemSample{At: r.now}, 10*time.Minute)
if got := r.sent(); len(got) != 4 {
t.Errorf("FAIL: a second clear sent again: %v", got)
}
if !t.Failed() {
t.Log("ok")
}
}
func TestOpsAlertFlapAndRestartWithinTheHourSayNothingMore(t *testing.T) {
r := newAlertRig("dana")
r.a.WarningStarted(warn(obs.WarnKindAnon), r.now)
r.now = r.now.Add(20 * time.Minute)
r.a.WarningCleared(warn(obs.WarnKindAnon), obs.MemSample{At: r.now}, 20*time.Minute)
r.now = r.now.Add(10 * time.Minute)
r.a.WarningStarted(warn(obs.WarnKindAnon), r.now) // flap, 30 min after the first
r.now = r.now.Add(10 * time.Minute)
r.a.WarningCleared(warn(obs.WarnKindAnon), obs.MemSample{At: r.now}, 10*time.Minute)
if got := r.sent(); len(got) != 2 {
t.Errorf("FAIL: a flap inside the hour sent: %v", got)
}
// A new process, same store: the condition is already true again an
// hour on from the last telling, so it is told; ten minutes after
// that it is not told again by a restart.
r.now = r.now.Add(time.Hour)
r.a.WarningStarted(warn(obs.WarnKindAnon), r.now)
restarted := &OpsAlerter{Deps: r.a.Deps, People: r.a.People, State: r.prefs, Now: r.a.Now}
r.now = r.now.Add(10 * time.Minute)
restarted.WarningStarted(warn(obs.WarnKindAnon), r.now)
if got := r.sent(); len(got) != 3 {
t.Errorf("FAIL: a restart with a recent telling sent again: %v", got)
}
// And its clear still reaches the person who was told of the start.
restarted.WarningCleared(warn(obs.WarnKindAnon), obs.MemSample{At: r.now}, time.Minute)
if got := r.sent(); len(got) != 4 {
t.Errorf("FAIL: clear after a restart was not sent: %v", got)
}
if !t.Failed() {
t.Log("ok")
}
}
func TestOpsAlertOnlyTheTableKindsAlert(t *testing.T) {
r := newAlertRig("dana")
for _, k := range []string{obs.WarnKindGrowth, obs.WarnKindSwap, "nonsense"} {
r.a.WarningStarted(warn(k), r.now)
r.a.WarningCleared(warn(k), obs.MemSample{At: r.now}, time.Minute)
}
if got := r.sent(); len(got) != 0 {
t.Errorf("FAIL: kinds outside the table alerted: %v", got)
}
for _, k := range []string{obs.WarnKindHost, obs.WarnKindAnon, obs.WarnKindCgroup} {
if _, ok := opsAlertKinds[k]; !ok {
t.Errorf("FAIL: %s is not in the table", k)
}
}
if len(opsAlertKinds) != 3 {
t.Errorf("FAIL: the table has %d kinds, want 3", len(opsAlertKinds))
}
if !t.Failed() {
t.Log("ok")
}
}
func TestOpsAlertNonLeaderSendsNothing(t *testing.T) {
r := newAlertRig("dana")
r.a.Leader = func() bool { return false }
r.a.WarningStarted(warn(obs.WarnKindHost), r.now)
r.a.WarningCleared(warn(obs.WarnKindHost), obs.MemSample{At: r.now}, time.Minute)
if got := r.sent(); len(got) != 0 {
t.Errorf("FAIL: a node that is not the leader sent: %v", got)
}
}
func TestOpsAlertPayloadCarriesTheData(t *testing.T) {
r := newAlertRig("dana")
r.a.WarningStarted(obs.MemWarning{Kind: obs.WarnKindAnon, Value: 7 << 30, Threshold: obs.WarnAnon}, r.now)
r.sent()
evs := r.fn.snapshot()
if len(evs) != 1 {
t.Fatalf("FAIL: %d events", len(evs))
}
p := evs[0].Payload
if p["kind"] != obs.WarnKindAnon || p["value"] != uint64(7<<30) || p["threshold"] != uint64(obs.WarnAnon) || p["phase"] != "start" {
t.Errorf("FAIL: payload data: %v", p)
}
for _, k := range []string{"title", "summary", "text", "link"} {
if s, _ := p[k].(string); s == "" {
t.Errorf("FAIL: payload lacks %s", k)
}
}
if evs[0].TxnID == "" {
t.Errorf("FAIL: no idempotency key")
}
// The same edge twice is one key.
r2 := newAlertRig("dana")
r2.a.State = nil
r2.a.WarningStarted(warn(obs.WarnKindHost), r.now)
r2.a.WarningStarted(warn(obs.WarnKindHost), r.now)
r2.sent()
e := r2.fn.snapshot()
if len(e) != 2 || e[0].TxnID != e[1].TxnID {
t.Errorf("FAIL: a double fire has distinct keys: %+v", e)
}
// The mail card is built, and never for a forge with no origin.
h, _ := (Deps{Origin: "https://forge.example"}).withMailHTML(evs[0], r.now).Payload["html"].(string)
if h == "" {
t.Errorf("FAIL: no mail card")
}
if !t.Failed() {
t.Log("ok")
}
}
func TestAdminPeopleExcludesAgentsDisabledAndOrdinary(t *testing.T) {
got := AdminPeople([]oltp.Account{
{Name: "dana", Admin: true},
{Name: "bot", Admin: true, Operator: "dana"},
{Name: "gone", Admin: true, Disabled: "2026-01-01"},
{Name: "ordinary"},
})
if len(got) != 1 || got[0] != "dana" {
t.Errorf("FAIL: recipients %v", got)
}
}
type failingNotify struct{}
func (failingNotify) Trigger(context.Context, NotifyEvent) error { return errors.New("down") }
func TestOpsAlertNotifyFailureIsSurvived(t *testing.T) {
a := &OpsAlerter{Deps: &Deps{Notify: failingNotify{}}, People: func() []string { return []string{"dana"} }, State: &memPrefs{}}
a.WarningStarted(warn(obs.WarnKindHost), time.Now())
time.Sleep(100 * time.Millisecond)
a.WarningCleared(warn(obs.WarnKindHost), obs.MemSample{At: time.Now()}, time.Minute)
time.Sleep(100 * time.Millisecond)
}
// The matrix has an operations row for a site admin and for nobody else.
func TestOperationsRowIsForSiteAdminsOnly(t *testing.T) {
d := Deps{CapsFor: func(who, _ string) CapSet {
if who == "dana" {
return oltp.CapSet("s")
}
return oltp.CapSet("r")
}}
has := func(user string) bool {
for _, f := range d.prefFamiliesFor(ops.WithUser(context.Background(), user)) {
if f.family == "operations" && f.workflow == "quarry-ops-alert" {
return true
}
}
return false
}
if !has("dana") {
t.Errorf("FAIL: a site admin has no operations row")
}
if has("eli") {
t.Errorf("FAIL: an ordinary account has an operations row")
}
if !t.Failed() {
t.Log("ok")
}
}
// The forge's own figure travels with a start, and a clear carries the
// numbers as they are now rather than as they were.
func TestOpsAlertPayloadHasTheForgeFigureAndCurrentNumbers(t *testing.T) {
r := newAlertRig("dana")
r.a.WarningStarted(obs.MemWarning{Kind: obs.WarnKindHost, Value: 1 << 30, Threshold: obs.WarnHostAvailable, Forge: 5 << 30}, r.now)
r.sent()
r.a.WarningCleared(obs.MemWarning{Kind: obs.WarnKindHost, Value: 1 << 30, Threshold: obs.WarnHostAvailable},
obs.MemSample{At: r.now.Add(time.Minute), Available: 9 << 30, Anon: 2 << 30, File: 1 << 30}, time.Minute)
r.sent()
evs := r.fn.snapshot()
if len(evs) != 2 {
t.Fatalf("FAIL: %d events", len(evs))
}
if evs[0].Payload["forge"] != uint64(5<<30) {
t.Errorf("FAIL: start forge figure: %v", evs[0].Payload["forge"])
}
if evs[1].Payload["value"] != uint64(9<<30) || evs[1].Payload["forge"] != uint64(3<<30) || evs[1].Payload["phase"] != "clear" {
t.Errorf("FAIL: clear payload: %v", evs[1].Payload)
}
if mailSurface[kindOpsAlert] == "" {
t.Errorf("FAIL: the kind has no header label in the table")
}
if !t.Failed() {
t.Log("ok")
}
}