package obs
import (
"sync"
"time"
)
// Signals are the four golden signals over the last hour, one bin a
// minute, oldest first: how many renders began, how many of them were
// over Budget, and the most store subprocesses that were running at
// once. Server errors are the fourth and live in the error journal,
// which keeps a row per occurrence and answers the same question itself.
//
// The rings behind the cards hold the last few hundred events, which on
// a quiet forge is hours and on a busy one is seconds, so a line drawn
// from them has no fixed scale and a three-minute window of them
// quantises to 0, 1 and 2. These bins are kept as they happen, bounded
// by the hour and not by traffic, so the line beside a number is an
// hour of that number whatever the forge is doing.
//
// Every series is always SignalMinutes long and aligned on End: the
// last bin is the minute starting at End, still filling. A minute no
// process was running in (before the first start, or while the forge
// was down between two) is not a quiet minute, and Up says so, so a
// chart can leave a gap instead of drawing zeros.
type Signals struct {
// Minutes is how many bins there are.
Minutes int `json:"minutes"`
// End is the start of the last bin, RFC3339 UTC.
End string `json:"end"`
Renders []int `json:"renders"`
Slow []int `json:"slow"`
Busy []int `json:"busy"`
// Up is whether a process of this forge was running in each minute.
Up []bool `json:"up"`
// Starts are the moments a process came up within the hour, oldest
// first, RFC3339 UTC: the deploys.
Starts []string `json:"starts,omitempty"`
}
// SignalMinutes is how far back the signals reach.
const SignalMinutes = 60
type signalRing struct {
mu sync.Mutex
saveMu sync.Mutex // serialises writes of the state file
at time.Time // the minute the last bin is for, truncated; zero until first use
rend [SignalMinutes]int
slow [SignalMinutes]int
busy [SignalMinutes]int
up [SignalMinutes]bool
starts []time.Time
}
var signals signalRing
// roll advances the ring to now's minute. The minutes that passed with
// nothing recorded in them are filled with zeros and marked live, so a
// quiet hour reads as quiet rather than as the last busy minute
// stretched; roll is only ever called by a running process, which is
// what makes those minutes live. Minutes the process was not alive for
// are marked down by restore, which advances with live=false instead.
// clockStepMinutes is how far behind the ring's newest minute the wall
// clock must read before it counts as a step back.
const clockStepMinutes = 2
func (s *signalRing) roll(now time.Time, live bool) {
m := now.Truncate(time.Minute)
if s.at.IsZero() {
s.at = m
s.up[SignalMinutes-1] = live
return
}
n := int(m.Sub(s.at) / time.Minute)
if n <= -clockStepMinutes {
// The clock stepped back. Carry on from the earlier minute rather
// than hold the ring at a minute that will not come for a while,
// which would put every count in a bin the page reads as the
// future: the bins keep their counts, the newest is now the
// minute the clock says, and a deploy dated after it is dropped.
s.at = m
cut := 0
for cut < len(s.starts) && !s.starts[cut].After(m.Add(time.Minute)) {
cut++
}
s.starts = s.starts[:cut]
return
}
if n < 0 {
// A caller that read the clock just before the minute turned and
// took the lock just after. Not a step: moving the ring back for
// it would shift the whole hour again on the next call.
return
}
if n == 0 {
return
}
s.at = m
if n >= SignalMinutes {
s.rend, s.slow, s.busy = [SignalMinutes]int{}, [SignalMinutes]int{}, [SignalMinutes]int{}
for i := range s.up {
s.up[i] = live
}
} else {
copy(s.rend[:], s.rend[n:])
copy(s.slow[:], s.slow[n:])
copy(s.busy[:], s.busy[n:])
copy(s.up[:], s.up[n:])
for i := SignalMinutes - n; i < SignalMinutes; i++ {
s.rend[i], s.slow[i], s.busy[i], s.up[i] = 0, 0, 0, live
}
}
s.trimStarts()
}
// trimStarts drops the deploys that fell out of the hour.
func (s *signalRing) trimStarts() {
cut := s.at.Add(-(SignalMinutes - 1) * time.Minute)
i := 0
for i < len(s.starts) && s.starts[i].Before(cut) {
i++
}
s.starts = s.starts[i:]
}
func (s *signalRing) render(now time.Time, slow bool) {
s.mu.Lock()
defer s.mu.Unlock()
// A render is recorded when it ends, under the time it began, so one
// that ran across a minute's turn arrives dated a minute the ring has
// already left. It is counted in that minute's bin; it never moves
// the ring, which only the wall clock does.
i := SignalMinutes - 1
if m := now.Truncate(time.Minute); !s.at.IsZero() && m.Before(s.at) {
i -= int(s.at.Sub(m) / time.Minute)
if i < 0 {
return
}
} else {
s.roll(now, true)
}
s.rend[i]++
if slow {
s.slow[i]++
}
}
func (s *signalRing) running(now time.Time, live int) {
s.mu.Lock()
defer s.mu.Unlock()
s.roll(now, true)
if live > s.busy[SignalMinutes-1] {
s.busy[SignalMinutes-1] = live
}
}
// snapshot is the hour as of now, the empty minutes since the last
// event filled in, so the line ends at now and not at the last thing
// that happened.
func (s *signalRing) snapshot(now time.Time) Signals {
s.mu.Lock()
defer s.mu.Unlock()
s.roll(now, true)
out := Signals{Minutes: SignalMinutes, End: s.at.UTC().Format(time.RFC3339),
Renders: append([]int(nil), s.rend[:]...),
Slow: append([]int(nil), s.slow[:]...),
Busy: append([]int(nil), s.busy[:]...),
Up: append([]bool(nil), s.up[:]...)}
for _, t := range s.starts {
out.Starts = append(out.Starts, t.Format(time.RFC3339))
}
return out
}
// SignalsNow is the last hour of signals, as of now.
func SignalsNow() Signals { return signals.snapshot(time.Now()) }
// resetSignals empties the ring, for tests.
func resetSignals() { signals = signalRing{} }