package obs
import (
"context"
"encoding/json"
"os"
"path/filepath"
"time"
)
// The hour behind the operations page's four charts survives a restart.
// Without it every deploy empties the charts, and the hour is mostly the
// previous process's, which is exactly the part worth seeing.
//
// What is kept is small and bounded by the hour: sixty bins of three
// counts, which minutes a process was running in, and the moments
// processes came up. It is one small file in the data directory, written
// once a minute and at shutdown, replaced whole so a crash mid-write
// leaves the last good one. It is derived data: a missing, unreadable or
// inconsistent file is an empty hour, never a failed start.
//
// The fourth signal, server errors, is not in the file; the error
// journal is already durable and answers for any minute.
// signalFileVersion is bumped when the file's meaning changes, so an old
// file is read as empty rather than misread.
const signalFileVersion = 1
type signalFile struct {
V int `json:"v"`
End string `json:"end"`
Renders []int `json:"renders"`
Slow []int `json:"slow"`
Busy []int `json:"busy"`
Up []bool `json:"up"`
Starts []string `json:"starts"`
}
// StartSignals loads the saved hour from path, notes that a process came
// up at now, and returns. The minutes between the file's last write and
// now are marked as no process running, which is what they were.
func StartSignals(path string, now time.Time) {
signals.restore(path, now)
}
// KeepSignals writes the hour to path every minute until ctx ends.
func KeepSignals(ctx context.Context, path string) {
go func() {
t := time.NewTicker(time.Minute)
defer t.Stop()
for {
select {
case <-ctx.Done():
return
case <-t.C:
SaveSignals(path)
}
}
}()
}
// SaveSignals writes the hour to path now, for shutdown. An error is
// dropped on purpose: the file is a convenience, and a full disk must
// not turn a clean stop into a failed one.
func SaveSignals(path string) {
signals.save(path, time.Now())
}
func (s *signalRing) save(path string, now time.Time) {
// One write at a time, from the snapshot to the rename: the minute's
// tick and the shutdown's save would otherwise race, and the older
// snapshot's rename could land last.
s.saveMu.Lock()
defer s.saveMu.Unlock()
s.mu.Lock()
s.roll(now, true)
f := signalFile{V: signalFileVersion, End: s.at.UTC().Format(time.RFC3339),
Renders: append([]int(nil), s.rend[:]...), Slow: append([]int(nil), s.slow[:]...),
Busy: append([]int(nil), s.busy[:]...), Up: append([]bool(nil), s.up[:]...)}
for _, t := range s.starts {
f.Starts = append(f.Starts, t.Format(time.RFC3339))
}
s.mu.Unlock()
b, err := json.Marshal(f)
if err != nil {
return
}
tmp, err := os.CreateTemp(filepath.Dir(path), ".ops-hour-*")
if err != nil {
return
}
_, werr := tmp.Write(b)
cerr := tmp.Close()
if werr != nil || cerr != nil || os.Rename(tmp.Name(), path) != nil {
os.Remove(tmp.Name())
}
}
func (s *signalRing) restore(path string, now time.Time) {
s.mu.Lock()
defer s.mu.Unlock()
s.at, s.rend, s.slow, s.busy, s.up, s.starts = time.Time{}, [SignalMinutes]int{}, [SignalMinutes]int{}, [SignalMinutes]int{}, [SignalMinutes]bool{}, nil
// A file that ends after now was written by a clock that has since
// stepped back; rolling forward from it would freeze every bin in a
// minute that has not come, so it is an empty hour.
if f, ok := readSignalFile(path); ok && !fileEnd(f).After(now.Truncate(time.Minute)) {
s.at = fileEnd(f)
copy(s.rend[:], f.Renders)
copy(s.slow[:], f.Slow)
copy(s.busy[:], f.Busy)
copy(s.up[:], f.Up)
for _, x := range f.Starts {
if t, err := time.Parse(time.RFC3339, x); err == nil {
s.starts = append(s.starts, t)
}
}
// The minutes between the last write and now had no process.
s.roll(now, false)
}
s.roll(now, true)
s.up[SignalMinutes-1] = true
s.starts = append(s.starts, now.UTC())
s.trimStarts()
}
func fileEnd(f signalFile) time.Time {
t, _ := time.Parse(time.RFC3339, f.End)
return t
}
// readSignalFile reads and checks the file; anything unexpected is
// "no file".
func readSignalFile(path string) (signalFile, bool) {
var f signalFile
b, err := os.ReadFile(path)
if err != nil || json.Unmarshal(b, &f) != nil || f.V != signalFileVersion {
return signalFile{}, false
}
if len(f.Renders) != SignalMinutes || len(f.Slow) != SignalMinutes ||
len(f.Busy) != SignalMinutes || len(f.Up) != SignalMinutes {
return signalFile{}, false
}
if _, err := time.Parse(time.RFC3339, f.End); err != nil {
return signalFile{}, false
}
for _, n := range [][]int{f.Renders, f.Slow, f.Busy} {
for _, v := range n {
if v < 0 {
return signalFile{}, false
}
}
}
return f, true
}