package obs
import (
"errors"
"fmt"
"strings"
"sync"
"testing"
"time"
)
const statusFixture = `Name: quarry
VmRSS: 9000000 kB
RssAnon: 6291456 kB
RssFile: 204800 kB
RssShmem: 0 kB
`
const meminfoFixture = `MemTotal: 16000000 kB
MemFree: 500000 kB
MemAvailable: 2097152 kB
SwapTotal: 4194304 kB
SwapFree: 419430 kB
`
func TestParseProc(t *testing.T) {
a, f := parseStatus(statusFixture)
if a != 6<<30 || f != 200<<20 {
t.Errorf("FAIL: status anon=%d file=%d", a, f)
}
a, f = parseStatus("VmRSS:\t 1024 kB\n")
if a != 1<<20 || f != 0 {
t.Errorf("FAIL: VmRSS fallback anon=%d file=%d", a, f)
}
av, sf, st := parseMeminfo(meminfoFixture)
if av != 2<<30 || sf != 419430<<10 || st != 4<<30 {
t.Errorf("FAIL: meminfo %d %d %d", av, sf, st)
}
if a, f := parseStatus("garbage\nRssAnon: x kB\n"); a != 0 || f != 0 {
t.Errorf("FAIL: garbage read as %d %d", a, f)
}
if !t.Failed() {
t.Log("ok")
}
}
// With no /proc every reading is zero and nothing warns.
func TestNoProcNoWarning(t *testing.T) {
old := procRead
defer func() { procRead = old }()
procRead = func(string) ([]byte, error) { return nil, errors.New("no /proc") }
if p := readProcMem(); p != (procMem{}) {
t.Errorf("FAIL: readings without /proc: %+v", p)
}
s := readMemory()
if s.Anon != 0 || s.Available != 0 || s.SwapTotal != 0 {
t.Errorf("FAIL: sample without /proc: %+v", s)
}
if w := Warnings([]MemSample{s, s}, 0); len(w) != 0 {
t.Errorf("FAIL: warnings from unknowns: %+v", w)
}
if w := Warnings([]MemSample{s}, 9<<30); len(w) != 0 {
t.Errorf("FAIL: cgroup warning from unknowns: %+v", w)
}
if !t.Failed() {
t.Log("ok")
}
}
func TestReadProcMemInjected(t *testing.T) {
old := procRead
defer func() { procRead = old }()
procRead = func(p string) ([]byte, error) {
switch p {
case "/proc/self/status":
return []byte(statusFixture), nil
case "/proc/meminfo":
return []byte(meminfoFixture), nil
}
return nil, errors.New("absent")
}
p := readProcMem()
if p.Anon != 6<<30 || p.Available != 2<<30 || p.SwapTotal != 4<<30 {
t.Errorf("FAIL: %+v", p)
}
}
func kinds(ws []MemWarning) string {
var k []string
for _, w := range ws {
k = append(k, w.Kind)
}
return strings.Join(k, ",")
}
// series builds one sample a minute ending at end, anon from f(minutes ago).
func series(end time.Time, mins int, f func(ago int) uint64) []MemSample {
var s []MemSample
for ago := mins; ago >= 0; ago-- {
s = append(s, MemSample{At: end.Add(-time.Duration(ago) * time.Minute), Anon: f(ago)})
}
return s
}
func TestWarnAnonWindow(t *testing.T) {
end := time.Now()
over := func(ago int) uint64 { return WarnAnon }
if w := Warnings(series(end, 10, over), 0); kinds(w) != WarnKindAnon {
t.Errorf("FAIL: at the line for 10 min: %q", kinds(w))
}
if w := Warnings(series(end, 9, over), 0); len(w) != 0 {
t.Errorf("FAIL: at the line for 9 min: %q", kinds(w))
}
below := func(ago int) uint64 { return WarnAnon - 1 }
if w := Warnings(series(end, 30, below), 0); len(w) != 0 {
t.Errorf("FAIL: one byte under: %q", kinds(w))
}
// A dip inside the window resets it.
dip := func(ago int) uint64 {
if ago == 5 {
return 1
}
return WarnAnon
}
if w := Warnings(series(end, 12, dip), 0); len(w) != 0 {
t.Errorf("FAIL: a dip 5 min ago still warned: %q", kinds(w))
}
if !t.Failed() {
t.Log("ok")
}
}
func TestWarnGrowthWindow(t *testing.T) {
end := time.Now()
// anon rises perHour bytes each hour, evenly, for mins minutes.
rise := func(perHour uint64) func(int) uint64 {
return func(ago int) uint64 { return 1<<30 + (1000-uint64(ago))*perHour/60 }
}
if w := Warnings(series(end, 180, rise(WarnGrowthPerHour)), 0); kinds(w) != WarnKindGrowth {
t.Errorf("FAIL: 300 MiB/h for 3 h: %q", kinds(w))
}
if w := Warnings(series(end, 180, rise(WarnGrowthPerHour-120)), 0); len(w) != 0 {
t.Errorf("FAIL: just under the rate: %q", kinds(w))
}
if w := Warnings(series(end, 150, rise(WarnGrowthPerHour)), 0); len(w) != 0 {
t.Errorf("FAIL: only 2.5 h of readings: %q", kinds(w))
}
// Fast for two hours, then flat: not sustained.
flat := func(ago int) uint64 {
if ago < 60 {
return rise(WarnGrowthPerHour)(60)
}
return rise(WarnGrowthPerHour)(ago)
}
if w := Warnings(series(end, 180, flat), 0); len(w) != 0 {
t.Errorf("FAIL: last hour flat still warned: %q", kinds(w))
}
// The day-long ring covers the window.
if MemoryKept < WarnGrowthFor {
t.Errorf("FAIL: ring keeps %s, rule needs %s", MemoryKept, WarnGrowthFor)
}
if !t.Failed() {
t.Log("ok")
}
}
func TestWarnHostSwapCgroup(t *testing.T) {
now := time.Now()
one := func(s MemSample, cap uint64) string { s.At = now; return kinds(Warnings([]MemSample{s}, cap)) }
if k := one(MemSample{Available: WarnHostAvailable}, 0); k != "" {
t.Errorf("FAIL: available at the line: %q", k)
}
if k := one(MemSample{Available: WarnHostAvailable - 1}, 0); k != WarnKindHost {
t.Errorf("FAIL: available under the line: %q", k)
}
total := uint64(1000 << 20)
if k := one(MemSample{SwapTotal: total, SwapFree: total / 10 * 2}, 0); k != "" {
t.Errorf("FAIL: swap 80%% used: %q", k)
}
if k := one(MemSample{SwapTotal: total, SwapFree: total / 10}, 0); k != WarnKindSwap {
t.Errorf("FAIL: swap 90%% used: %q", k)
}
if k := one(MemSample{SwapTotal: 0, SwapFree: 0}, 0); k != "" {
t.Errorf("FAIL: no swap configured: %q", k)
}
capb := uint64(10 << 30)
if k := one(MemSample{Anon: capb/10*8 - 1}, capb); k != "" {
t.Errorf("FAIL: under 80%% of the cap: %q", k)
}
if k := one(MemSample{Anon: capb / 10 * 8}, capb); k != WarnKindCgroup {
t.Errorf("FAIL: at 80%% of the cap: %q", k)
}
if k := one(MemSample{Anon: capb}, 0); k != "" {
t.Errorf("FAIL: no cap set: %q", k)
}
if !t.Failed() {
t.Log("ok")
}
}
// One line when a warning starts, one when it clears, none between.
func TestWatchLogsEdges(t *testing.T) {
var lines []string
w := memWatch{logf: func(f string, a ...any) { lines = append(lines, fmt.Sprintf(f, a...)) }}
now := time.Now()
on := []MemWarning{{Kind: WarnKindHost, Value: 1, Threshold: 2}}
w.observe(MemSample{At: now}, nil)
w.observe(MemSample{At: now.Add(time.Minute)}, on)
w.observe(MemSample{At: now.Add(2 * time.Minute)}, on)
w.observe(MemSample{At: now.Add(3 * time.Minute)}, on)
if len(lines) != 1 || !strings.Contains(lines[0], WarnKindHost) {
t.Fatalf("FAIL: want one start line, got %q", lines)
}
w.observe(MemSample{At: now.Add(4 * time.Minute)}, nil)
w.observe(MemSample{At: now.Add(5 * time.Minute)}, nil)
if len(lines) != 2 || !strings.Contains(lines[1], WarnKindHost) {
t.Fatalf("FAIL: want a clear line, got %q", lines)
}
if !t.Failed() {
t.Log("ok")
}
}
func TestMemoryNowCarriesWarningsField(t *testing.T) {
if m := MemoryNow(); m.Warnings == nil {
t.Error("FAIL: Warnings is nil, which marshals as null")
}
}
type recAlerter struct {
mu sync.Mutex
started []string
cleared []string
panicOn string
block chan struct{}
}
func (a *recAlerter) WarningStarted(w MemWarning, _ time.Time) {
if a.block != nil {
<-a.block
}
if w.Kind == a.panicOn {
panic("notifier broke")
}
a.mu.Lock()
a.started = append(a.started, w.Kind)
a.mu.Unlock()
}
func (a *recAlerter) WarningCleared(w MemWarning, _ MemSample, _ time.Duration) {
a.mu.Lock()
a.cleared = append(a.cleared, w.Kind)
a.mu.Unlock()
}
// The alerter hears each edge once, never per sample, and a notifier
// that panics or hangs does not stop the sampler.
func TestWatchAlerterHearsEdgesOnlyAndCannotStopSampling(t *testing.T) {
rec := &recAlerter{panicOn: WarnKindSwap}
w := memWatch{logf: func(string, ...any) {}, alert: rec}
now := time.Now()
host := []MemWarning{{Kind: WarnKindHost}}
for i := 0; i < 5; i++ {
w.observe(MemSample{At: now.Add(time.Duration(i) * time.Minute)}, host)
}
w.observe(MemSample{At: now.Add(6 * time.Minute)}, []MemWarning{{Kind: WarnKindSwap}}) // starts swap, clears host
w.observe(MemSample{At: now.Add(7 * time.Minute)}, nil) // clears swap
w.wait()
if len(rec.started) != 1 || rec.started[0] != WarnKindHost || len(rec.cleared) != 2 {
t.Errorf("FAIL: started %v cleared %v", rec.started, rec.cleared)
}
// A hung notifier: observe returns at once and drops what overflows.
hung := &recAlerter{block: make(chan struct{})}
w2 := memWatch{logf: func(string, ...any) {}, alert: hung}
done := make(chan struct{})
go func() {
for i := 0; i < alertQueue*3; i++ {
w2.observe(MemSample{At: now}, []MemWarning{{Kind: fmt.Sprint("k", i)}})
}
close(done)
}()
select {
case <-done:
case <-time.After(5 * time.Second):
t.Fatal("FAIL: a hung alerter blocked the sampler")
}
close(hung.block)
w2.wait()
if !t.Failed() {
t.Log("ok")
}
}
// A warning carries what the forge itself holds, so a reader can tell a
// server short of memory because of the forge from one short of it for
// another reason.
func TestWarningCarriesTheForgesOwnFigure(t *testing.T) {
s := MemSample{At: time.Now(), Anon: 3 << 30, File: 1 << 30, Available: WarnHostAvailable - 1}
w := Warnings([]MemSample{s}, 0)
if len(w) != 1 || w[0].Forge != 4<<30 {
t.Errorf("FAIL: %+v", w)
}
s.Anon, s.File = 0, 0
if w := Warnings([]MemSample{s}, 0); len(w) != 1 || w[0].Forge != 0 {
t.Errorf("FAIL: unknown forge figure read as %+v", w)
}
}