// Package health watches the services the forge cannot do its job
// without — the CI control plane and the notification companion — and
// makes their absence loud.
//
// The failure this exists to prevent: chat sat down for most of a day
// and every CI run failed, and nobody was told. Each individual feature
// degraded politely and silently, which is the correct behaviour for a
// request and the wrong behaviour for a fleet.
//
// Two rules shape the design:
//
// 1. The alarm must not travel through the thing that is broken. A
// notifier that is down cannot tell you it is down, so a transition
// is written to the error journal FIRST — which is local, durable
// and already the panel a maintainer reads — and only then offered
// to the notifier as a best effort.
//
// 2. State changes are the event, not states. Probing every thirty
// seconds and reporting every failure would file 2,880 identical
// reports a day; a service that went down once produces exactly one
// report, and one more when it comes back.
package health
import (
"context"
"fmt"
"net/http"
"sync"
"time"
)
// Dep is one service the forge depends on.
type Dep struct {
// Name is what a maintainer calls it: "dagnats", "notify".
Name string
// URL is probed. A dependency with no URL is not configured, and an
// unconfigured dependency is not a fault — it is a choice.
URL string
// Why says, in a maintainer's terms, what stops working when this
// is down. An alert that names a hostname teaches nothing; one that
// says "CI cannot run" is actionable.
Why string
}
// State is what the watcher currently believes about one dependency.
type State struct {
Dep
Up bool
// Since is when it entered this state — "down for 3h" is the number
// that makes somebody act.
Since time.Time
// Err is the last probe failure, empty when up.
Err string
// Checked is when the last probe ran, so the panel can say whether
// this belief is current or stale.
Checked time.Time
}
// Down reports how long this dependency has been down, zero when up.
func (s State) Down() time.Duration {
if s.Up {
return 0
}
return time.Since(s.Since)
}
// DownFor is how long it has been down, in a maintainer's words. The
// number that makes somebody act is "3h", not a timestamp.
func (s State) DownFor() string {
d := s.Down()
switch {
case d <= 0:
return ""
case d < time.Minute:
return fmt.Sprintf("%ds", int(d.Seconds()))
case d < time.Hour:
return fmt.Sprintf("%dm", int(d.Minutes()))
case d < 48*time.Hour:
return fmt.Sprintf("%dh%dm", int(d.Hours()), int(d.Minutes())%60)
}
return fmt.Sprintf("%dd", int(d.Hours()/24))
}
// Watch probes a set of dependencies on a fixed beat.
type Watch struct {
mu sync.Mutex
states map[string]*State
order []string
client *http.Client
// report receives one line per TRANSITION. Wired to the error
// journal, which is local and durable — see rule 1 above.
report func(dep Dep, up bool, err error)
}
// New builds a watcher over the configured dependencies. Dependencies
// with an empty URL are skipped: not configured is not broken.
func New(deps []Dep, report func(dep Dep, up bool, err error)) *Watch {
w := &Watch{
states: map[string]*State{},
client: &http.Client{Timeout: 5 * time.Second},
report: report,
}
now := time.Now()
for _, d := range deps {
if d.URL == "" {
continue
}
// Assumed UP at birth, so a forge that starts healthy files
// nothing. The first failed probe is the first event.
w.states[d.Name] = &State{Dep: d, Up: true, Since: now}
w.order = append(w.order, d.Name)
}
return w
}
// Run probes on every tick until ctx is done. It probes once
// immediately: a forge that starts while its CI plane is down should
// say so at once, not in thirty seconds.
func (w *Watch) Run(ctx context.Context, every time.Duration) {
if len(w.order) == 0 {
return
}
if every <= 0 {
every = 30 * time.Second
}
w.Check(ctx)
t := time.NewTicker(every)
defer t.Stop()
for {
select {
case <-ctx.Done():
return
case <-t.C:
w.Check(ctx)
}
}
}
// Check probes every dependency once, in parallel: a hung service must
// not delay the verdict on a healthy one.
func (w *Watch) Check(ctx context.Context) {
var wg sync.WaitGroup
for _, name := range w.order {
w.mu.Lock()
st := w.states[name]
w.mu.Unlock()
wg.Add(1)
go func(d Dep) {
defer wg.Done()
w.observe(d, w.probe(ctx, d))
}(st.Dep)
}
wg.Wait()
}
// probe asks for the URL and cares only whether the service ANSWERED.
// Any HTTP status is an answer: a 404 or a redirect means the process
// is up and listening, which is the question. Only a transport failure
// — refused, unresolved, timed out — is "down".
func (w *Watch) probe(ctx context.Context, d Dep) error {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, d.URL, nil)
if err != nil {
return err
}
resp, err := w.client.Do(req)
if err != nil {
return err
}
resp.Body.Close()
// 5xx is the exception: the process answered, but answered that it
// is broken. For a dependency that is the same as being down.
if resp.StatusCode >= 500 {
return fmt.Errorf("%s", resp.Status)
}
return nil
}
// observe records one probe result and reports only a CHANGE.
func (w *Watch) observe(d Dep, err error) {
now := time.Now()
up := err == nil
w.mu.Lock()
st, ok := w.states[d.Name]
if !ok {
w.mu.Unlock()
return
}
st.Checked = now
if up {
st.Err = ""
} else {
st.Err = err.Error()
}
changed := st.Up != up
if changed {
st.Up, st.Since = up, now
}
w.mu.Unlock()
if changed && w.report != nil {
w.report(d, up, err)
}
}
// States returns the current beliefs, in the order the deps were given
// so the panel does not reshuffle between refreshes.
func (w *Watch) States() []State {
w.mu.Lock()
defer w.mu.Unlock()
out := make([]State, 0, len(w.order))
for _, name := range w.order {
out = append(out, *w.states[name])
}
return out
}
// Down returns just the dependencies currently failing — what a banner
// needs, and empty when there is nothing to say.
func (w *Watch) Down() []State {
var out []State
for _, s := range w.States() {
if !s.Up {
out = append(out, s)
}
}
return out
}