package api
import (
"fmt"
"log"
"strconv"
"time"
"craftdesign.group/quarry/internal/obs"
"craftdesign.group/quarry/internal/oltp"
)
// An operational alert is the forge telling its site admins, directly,
// that something needs them now: by the inbox and push at once and by
// email, as a sign-off request does. It is sent when a condition starts
// and once when it clears, never per reading. Adding a condition is one
// line in opsAlertKinds.
// opsAlertKind is how one kind of condition is worded. The numbers and
// their threshold travel as data in the payload; the words are the
// forge's own, since no caller wrote any.
type opsAlertKind struct {
// Title is the notification's heading and the email's subject.
Title string
// Says is what is wrong, from the number, the line it crossed and
// what the forge itself holds (0 when unknown).
Says func(value, threshold, forge uint64) string
// Next is what to do, said after Says.
Next string
// Now is the number a clear reports, read from the reading it
// cleared at, and Cleared words it.
Now func(s obs.MemSample) uint64
Cleared func(now, forge uint64) string
}
// forgeHolds says how much of the server's memory the forge holds.
func forgeHolds(forge uint64) string {
if forge == 0 {
return ""
}
return fmt.Sprintf(" The forge itself holds %s of it.", gb(forge))
}
// opsAlertKinds is the table of conditions that alert. A memory warning
// not listed here (growth, swap) stays on the operations page and in the
// journal.
var opsAlertKinds = map[string]opsAlertKind{
obs.WarnKindHost: {
Title: "The server is low on memory",
Says: func(v, t, forge uint64) string {
return fmt.Sprintf("The server has %s of memory free; it starts stopping programs below %s.", gb(v), gb(t)) + forgeHolds(forge)
},
// The forge may or may not be the cause, so a restart is offered
// only as what frees the forge's own share.
Next: "Open the operations page to see what is using it; restarting the forge frees only the forge's own share.",
Now: func(s obs.MemSample) uint64 { return s.Available },
Cleared: func(v, forge uint64) string {
return fmt.Sprintf("The server has %s of memory free again.", gb(v)) + forgeHolds(forge)
},
},
obs.WarnKindAnon: {
Title: "The forge is using a lot of memory",
Says: func(v, t, _ uint64) string {
return fmt.Sprintf("The forge is using %s of memory and has been over %s for ten minutes.", gb(v), gb(t))
},
Next: "Open the operations page; a restart of the forge clears it.",
Now: func(s obs.MemSample) uint64 { return s.Anon + s.File },
Cleared: func(v, _ uint64) string {
return fmt.Sprintf("The forge is using %s of memory now.", gb(v))
},
},
obs.WarnKindCgroup: {
Title: "The forge is close to its memory limit",
Says: func(v, t, _ uint64) string {
return fmt.Sprintf("The forge is using %s of memory; its limit stops it at 80%% of that, %s.", gb(v), gb(t))
},
Next: "Open the operations page; a restart of the forge clears it.",
Now: func(s obs.MemSample) uint64 { return s.Anon + s.File },
Cleared: func(v, _ uint64) string {
return fmt.Sprintf("The forge is using %s of memory now.", gb(v))
},
},
}
// opsAlertLink is the page that opens the condition up.
const opsAlertLink = "/fleet/ops#memory"
// opsAlertAgain is how long a person is left alone after being told
// about a kind: a condition that clears and starts again inside it, or
// that was already true when the forge restarted, says nothing more.
const opsAlertAgain = time.Hour
// OpsAlerter tells site admins who are people about operational
// conditions. The zero value tells nobody.
type OpsAlerter struct {
// Deps carries fireNotify, the forge's origin and its link maker.
Deps *Deps
// People are the accounts to tell: every open site admin who is a
// person, never an agent (AdminPeople).
People func() []string
// State remembers per person what they were told, so a restart or a
// flap does not tell them again. nil remembers nothing in memory
// beyond the process, which is a forge with no accounts store.
State AccountPrefStore
// Leader says this node does fleet-wide work; a node that is not it
// sends nothing, so a fleet sends once.
Leader func() bool
// Now is the clock; nil is time.Now.
Now func() time.Time
}
func (a *OpsAlerter) now() time.Time {
if a.Now != nil {
return a.Now()
}
return time.Now()
}
// AdminPeople is the open site-admin accounts that are people.
func AdminPeople(accounts []oltp.Account) []string {
var out []string
for _, ac := range accounts {
if ac.Admin && ac.Disabled == "" && !ac.Agent() {
out = append(out, ac.Name)
}
}
return out
}
// Keys in the accounts store, per person: when they were last told of a
// kind, and whether a start they were told of has not cleared.
func lastKey(kind string) string { return "opsalert." + kind }
func openKey(kind string) string { return "opsalert." + kind + ".open" }
// WarningStarted and WarningCleared make OpsAlerter the obs sampler's
// Alerter.
func (a *OpsAlerter) WarningStarted(w obs.MemWarning, at time.Time) {
a.start(w.Kind, w.Value, w.Threshold, w.Forge, at)
}
func (a *OpsAlerter) WarningCleared(was obs.MemWarning, now obs.MemSample, after time.Duration) {
a.clear(was.Kind, was.Threshold, now, after)
}
func (a *OpsAlerter) start(kind string, value, threshold, forge uint64, at time.Time) {
k, ok := opsAlertKinds[kind]
if !ok || !a.leads() {
return
}
for _, who := range a.people() {
if a.State != nil {
// Told within the hour: a flap, or a restart with the
// condition still true.
if t, err := time.Parse(time.RFC3339, a.State.AccountPref(who, lastKey(kind))); err == nil && a.now().Sub(t) < opsAlertAgain {
continue
}
}
a.tell(who, "start", kind, k.Title, k.Says(value, threshold, forge)+" "+k.Next, value, threshold, forge, at)
a.remember(who, kind, true)
}
}
func (a *OpsAlerter) clear(kind string, threshold uint64, cur obs.MemSample, after time.Duration) {
k, ok := opsAlertKinds[kind]
if !ok || !a.leads() {
return
}
value, forge := k.Now(cur), cur.Anon+cur.File
for _, who := range a.people() {
if a.State != nil && a.State.AccountPref(who, openKey(kind)) == "" {
continue // never told of the start, so nothing to take back
}
a.tell(who, "clear", kind, "Cleared: "+k.Title,
k.Cleared(value, forge)+fmt.Sprintf(" It lasted %d minutes. Nothing needs doing; the operations page has the history.", max(1, int(after.Round(time.Minute)/time.Minute))),
value, threshold, forge, cur.At)
if a.State != nil {
if err := a.State.SetAccountPref(who, openKey(kind), ""); err != nil {
log.Printf("opsalert: %s: %v", who, err)
}
}
}
}
func (a *OpsAlerter) leads() bool { return a.Leader == nil || a.Leader() }
// test sends who one alert marked as a test, through tell, so it proves
// the workflow, the push and the mail a real alert uses. It records
// nothing: a test is not a telling, and does not quiet a real alert.
func (a *OpsAlerter) test(who string) {
a.tell(who, "test", "test", "Test: operational alerts reach you",
"This is a test of the forge's operational alerts, sent on request. Nothing is wrong. A real alert tells you when the server is low on memory or the forge holds too much, once when it starts and once when it clears.",
0, 0, 0, a.now())
}
func (a *OpsAlerter) people() []string {
if a.People == nil {
return nil
}
return a.People()
}
// remember records that who was told of kind now and has an open one.
func (a *OpsAlerter) remember(who, kind string, open bool) {
if a.State == nil {
return
}
if err := a.State.SetAccountPref(who, lastKey(kind), a.now().UTC().Format(time.RFC3339)); err != nil {
log.Printf("opsalert: %s: %v", who, err)
}
if open {
if err := a.State.SetAccountPref(who, openKey(kind), "1"); err != nil {
log.Printf("opsalert: %s: %v", who, err)
}
}
}
// tell sends one alert to one person. The idempotency key names kind,
// phase, the reading's time and the person, so a double fire of one edge
// is one delivery.
func (a *OpsAlerter) tell(who, phase, kind, title, sentence string, value, threshold, forge uint64, at time.Time) {
if a.Deps == nil {
return
}
p := map[string]any{
"kind": kind, "phase": phase, "value": value, "threshold": threshold, "forge": forge,
"title": title, "summary": sentence, "text": sentence, "surface": "operations",
"link": a.Deps.link(opsAlertLink),
}
a.Deps.fireNotify(NotifyEvent{Workflow: "quarry-ops-alert", To: who, Payload: p,
TxnID: "opsalert:" + kind + ":" + phase + ":" + strconv.FormatInt(at.Unix(), 10) + ":" + who})
}