package api
import (
"fmt"
"time"
"craftdesign.group/quarry/internal/obs"
"craftdesign.group/quarry/internal/zitcli"
)
// opsMemory is obs.Memory with what the forge can account for: the
// parts of the heap it measures. The rest is "not accounted", and a
// large rest is what says leak rather than busy.
type opsMemory struct {
obs.Memory
Accounted []opsMemoryPart `json:"accounted"`
// Unaccounted is the heap minus every accounted part, 0 when the
// parts exceed it (they are measured at different moments).
Unaccounted uint64 `json:"unaccounted"`
}
type opsMemoryPart struct {
Name string `json:"name"`
Bytes uint64 `json:"bytes"`
Limit uint64 `json:"limit,omitempty"`
}
func opsMemoryNow() opsMemory {
m := opsMemory{Memory: obs.MemoryNow()}
used, limit := zitcli.ForgeCacheUse()
if used > 0 || limit > 0 {
m.Accounted = append(m.Accounted, opsMemoryPart{Name: "store caches", Bytes: uint64(max(used, 0)), Limit: uint64(max(limit, 0))})
}
var sum uint64
for _, p := range m.Accounted {
sum += p.Bytes
}
if m.Heap > sum {
m.Unaccounted = m.Heap - sum
}
return m
}
// swapWorry is the swap that is a finding on its own: past it, pages of
// the heap are being read back from disk on the request path.
const swapWorry = 1 << 30
// memoryFindings is the memory half of the verdict. A heap over the
// target is already a fault, before the cap: the collector then runs
// almost without pause and the heap still does not come down.
func memoryFindings(m opsMemory) []opsFinding {
var out []opsFinding
over := m.Limit > 0 && m.Heap > m.Limit
nearCap := m.Cap > 0 && m.Heap > m.Cap/10*9
if over || nearCap {
text := "heap " + gb(m.Heap)
if m.Limit > 0 {
text += fmt.Sprintf(", %.1f× the %s target", float64(m.Heap)/float64(m.Limit), gb(m.Limit))
}
if m.Cap > 0 {
text += ", cap " + gb(m.Cap)
}
if !m.OverSince.IsZero() {
text += ", over the target for " + since(m.OverSince)
}
if m.Swap > 0 {
text += "; " + gb(m.Swap) + " swapped"
}
text += fmt.Sprintf("; %s of it not accounted for", gb(m.Unaccounted))
out = append(out, opsFinding{About: "memory", Severity: "bad", Text: text})
} else if m.Swap > swapWorry {
out = append(out, opsFinding{About: "memory", Severity: "warn",
Text: gb(m.Swap) + " of the heap swapped out while it is under its target"})
}
for _, w := range m.Warnings {
if f, ok := warningFinding(w); ok {
out = append(out, f)
}
}
return out
}
// warningFinding words one obs.MemWarning as a finding of its own, with
// a key per kind so each opens and closes by itself.
func warningFinding(w obs.MemWarning) (opsFinding, bool) {
f := opsFinding{Key: "memory:" + w.Kind, About: "memory", Severity: "bad", Link: "/fleet/ops#memory"}
switch w.Kind {
case obs.WarnKindAnon:
f.Text = fmt.Sprintf("the forge is using %s of memory, over %s for %s; the Go heap does not show this, database connections do",
gb(w.Value), gb(w.Threshold), since(w.Since))
f.Figure, f.Note = gb(w.Value), "in use"
case obs.WarnKindGrowth:
f.Severity = "warn"
f.Text = fmt.Sprintf("the forge's memory has grown by %s or more every hour for %d hours",
gb(w.Threshold), int(obs.WarnGrowthFor/time.Hour))
f.Figure, f.Note = gb(w.Value)+"/h", "growth"
case obs.WarnKindHost:
f.Text = fmt.Sprintf("the host has %s of memory available, under %s", gb(w.Value), gb(w.Threshold))
f.Figure, f.Note = gb(w.Value), "available"
case obs.WarnKindSwap:
f.Text = fmt.Sprintf("host swap is %.0f%% used", float64(w.Value)/10)
f.Figure, f.Note = fmt.Sprintf("%.0f%%", float64(w.Value)/10), "swap used"
case obs.WarnKindCgroup:
f.Text = fmt.Sprintf("the forge is using %s of memory, past %s, 80%% of the limit set for it", gb(w.Value), gb(w.Threshold))
f.Figure, f.Note = gb(w.Value), "of the cap"
default:
return opsFinding{}, false
}
f.weight = float64(w.Value)
return f, true
}
func gb(b uint64) string {
switch {
case b >= 1<<30:
return fmt.Sprintf("%.1f GB", float64(b)/(1<<30))
case b >= 1<<20:
return fmt.Sprintf("%.0f MB", float64(b)/(1<<20))
default:
return fmt.Sprintf("%d KB", b>>10)
}
}
func since(t time.Time) string {
d := time.Since(t).Round(time.Minute)
if d >= time.Hour {
return fmt.Sprintf("%dh%02dm", int(d.Hours()), int(d.Minutes())%60)
}
return fmt.Sprintf("%dm", int(d.Minutes()))
}