package web
import (
_ "embed"
"encoding/json"
"fmt"
"html/template"
"math"
"net/url"
"sort"
"strconv"
"strings"
"time"
)
// The hour behind the operations page's four tiles, drawn by the server.
//
// Every chart starts at zero, says its top, and shares one clock: sixty
// one-minute slots ending now, in UTC like the rest of the page. Rates (renders,
// store calls running) are a line, counts (slow renders, server errors)
// are a bar a minute, because a count per minute has no slope. A restart
// of the forge is a dashed line labelled deploy, and the minutes no
// process was running in are a gap, never a run of zeros.
//
// Two views of the same numbers are always in the page: the four tiles,
// and one panel with the four on a shared axis. The page's data-view
// attribute picks which is shown, which is what makes the toggle work
// with no script: the control is a form button that reloads with ?hour=,
// and ophour.js only flips the attribute in place and reads a minute off
// the panel under the pointer.
//go:embed assets/ophour.js
var opsHourJS []byte
var (
opsHourOut = compactJS("ophour.js", opsHourJS)
opsHourURL = hashedAsset("ophour", opsHourOut)
)
// hourW and hourH are the chart's drawing box. A slot is hourW/N wide;
// the box is stretched to whatever the page gives it, so every stroke is
// non-scaling and no text is drawn inside it.
const (
hourW = 600.0
hourH = 100.0
)
// hourKeys are the four measures, in the order the strip draws them.
var hourKeys = []string{"renders", "slow", "errors", "busy"}
// opsHour is the hour ready to draw.
type opsHour struct {
// NoErrors is whether the forge sent no error series (no journal, or
// it could not be read): the errors measure is then unavailable, not
// an hour of zeros.
NoErrors bool
N int
First time.Time // the start of slot 0; zero when the page has no clock
Series map[string][]int
Up []bool
// Deploys are the restarts inside the hour, by slot position.
Deploys []hourDeploy
Limit int // the store-call lane's bound, 0 when unknown
// Now holds each measure's figure beside its chart.
Now map[string]int
}
type hourDeploy struct {
Pos float64 // in slots from the left edge, fractional
Minute int // the slot it falls in
At string // clock, UTC
}
// Pct is the position as a percentage of the plot's width.
func (d hourDeploy) Pct(n int) float64 { return d.Pos / float64(n) * 100 }
// hourKind says how a measure is drawn.
func hourBars(key string) bool { return key == "slow" || key == "errors" }
func hourName(key string) string {
switch key {
case "renders":
return "renders per minute"
case "slow":
return "routes over budget"
case "errors":
return "server errors"
}
return "store calls in flight"
}
// hourShort names a measure inside a readout, where the line carries
// all four and the full names would not fit on a phone.
func hourShort(key string) string {
switch key {
case "renders":
return "renders"
case "slow":
return "over budget"
case "errors":
return "errors"
}
return "store calls"
}
// HourHint says what the chart beside the number draws, which for the
// routes tile is not the number itself (that is routes, this is renders).
func (v opsView) HourHint(key string) string {
switch key {
case "renders":
return "renders begun each minute"
case "slow":
return "renders over budget each minute"
case "errors":
return "server errors each minute"
}
return "most store calls running at once"
}
// HourView builds the hour from what ops.status sent.
func (v opsView) HourView() *opsHour {
s := v.Signals
h := &opsHour{N: len(s.Renders), Limit: v.StoreBound(),
Series: map[string][]int{}, Now: map[string]int{}}
fit := func(xs []int) []int {
out := make([]int, h.N)
copy(out[h.N-min(len(xs), h.N):], xs[max(0, len(xs)-h.N):])
return out
}
h.Series["renders"], h.Series["slow"], h.Series["busy"] = fit(s.Renders), fit(s.Slow), fit(s.Busy)
h.Series["errors"] = fit(s.Errors)
h.NoErrors = len(s.Errors) == 0
h.Up = make([]bool, h.N)
for i := range h.Up {
h.Up[i] = true
}
if len(s.Up) == h.N {
copy(h.Up, s.Up)
}
if end, err := time.Parse(time.RFC3339, s.End); err == nil && h.N > 0 {
h.First = end.Add(-time.Duration(h.N-1) * time.Minute)
for _, x := range s.Starts {
t, err := time.Parse(time.RFC3339, x)
if err != nil {
continue
}
pos := float64(t.Sub(h.First)) / float64(time.Minute)
if pos < 0 || pos >= float64(h.N) {
continue
}
h.Deploys = append(h.Deploys, hourDeploy{Pos: pos, Minute: int(pos), At: hm(t)})
}
sort.Slice(h.Deploys, func(i, j int) bool { return h.Deploys[i].Pos < h.Deploys[j].Pos })
}
// The figure is the line's last point: the last whole minute, or
// nothing when no process was running in it (just after a restart),
// never a number the chart does not show.
h.Now["renders"] = -1
if h.N >= 2 && h.Up[h.N-2] {
h.Now["renders"] = h.Series["renders"][h.N-2]
}
h.Now["slow"] = v.RoutesOver
h.Now["errors"] = v.ErrorsLastHour()
if h.NoErrors {
h.Now["errors"] = -1
}
h.Now["busy"] = v.LiveSubprocess
return h
}
// RendersFigure is the renders tile's number: the line's last point, or
// a dash where there is none.
func (h *opsHour) RendersFigure() string { return h.figure("renders") }
// ErrorsFigure is the errors tile's number, a dash where the forge sent
// no series to draw it from.
func (h *opsHour) ErrorsFigure() string { return h.figure("errors") }
func (h *opsHour) figure(key string) string {
if h.Now[key] < 0 {
return "\u2014"
}
return strconv.Itoa(h.Now[key])
}
func hm(t time.Time) string { return t.UTC().Format("15:04") }
// slotAt is the clock at the start of slot i, "" when there is none.
func (h *opsHour) slotAt(i int) string {
if h.First.IsZero() {
return ""
}
return hm(h.First.Add(time.Duration(i) * time.Minute))
}
// last is the final slot a measure draws. The current minute is still
// filling, so a rate through it would dip for no reason and the line
// would end below the number beside it: renders stop at the last whole
// minute, the same one the figure is.
func (h *opsHour) last(key string) int {
if key == "renders" {
return h.N - 2
}
return h.N - 1
}
// peak is the largest value a measure drew and the slot it was first in.
func (h *opsHour) peak(key string) (v, at int) {
at = -1
for i := 0; i <= h.last(key); i++ {
if h.Up[i] && h.Series[key][i] > v {
v, at = h.Series[key][i], i
}
}
return v, at
}
// top is the figure the y axis reaches: the next of 1, 2, 5 times a
// power of ten above the largest value, or the limit where there is one,
// so how close a peak came to it is the picture.
func (h *opsHour) top(key string) float64 {
p, _ := h.peak(key)
t := hourTop(float64(p))
if key == "busy" && h.Limit > 0 && p <= h.Limit {
t = float64(h.Limit)
}
return t
}
func hourTop(max float64) float64 {
if max <= 1 {
return 1
}
e := math.Pow(10, math.Floor(math.Log10(max)))
for _, m := range []float64{1, 2, 5, 10} {
if m*e >= max {
return m * e
}
}
return 10 * e
}
func hourNum(f float64) string { return strconv.FormatFloat(f, 'f', -1, 64) }
// firstUp is the first minute a process was running in.
func (h *opsHour) firstUp() int {
for i, up := range h.Up {
if up {
return i
}
}
return h.N
}
// gaps are the runs of minutes with no process after the first one that
// had: the forge was down. The hour before the first process is simply
// not drawn.
func (h *opsHour) gaps() [][2]int {
var out [][2]int
for i := h.firstUp(); i < h.N; i++ {
if h.Up[i] {
continue
}
j := i
for j < h.N && !h.Up[j] {
j++
}
out = append(out, [2]int{i, j})
i = j
}
return out
}
// ticks are the x-axis slots: every fifteen minutes on the shared panel
// (three of them marked optional for a narrow screen), the two ends and
// the middle on a tile.
func (h *opsHour) ticks(five bool) []int {
if h.N < 2 {
return nil
}
if !five {
return []int{0, (h.N - 1) / 2, h.N - 1}
}
// Counted back from now in quarters of an hour, so the labels read
// as "15 minutes ago" would; the first is the left edge.
q := 15
if h.N < 46 {
q = (h.N - 1) / 4
}
last := h.N - 1
return []int{0, last - 3*q, last - 2*q, last - q, last}
}
func (h *opsHour) y(v, top float64) float64 {
if v > top {
v = top
}
return hourH - hourH*v/top
}
// label states the finding in words, for a screen reader and for the
// reader who hovers: the figure now, the peak, and what the deploy and
// the downtime did to the hour.
func (h *opsHour) label(key string) string {
var b strings.Builder
if key == "errors" && h.NoErrors {
return "server errors each minute, last hour: unavailable, the error journal was not read"
}
switch key {
case "slow":
fmt.Fprintf(&b, "renders over budget each minute, last hour: %d routes over now", h.Now[key])
case "errors":
fmt.Fprintf(&b, "server errors each minute, last hour: %d in the hour", h.Now[key])
default:
if h.Now[key] < 0 {
fmt.Fprintf(&b, "%s, last hour: no data for the last whole minute", hourName(key))
} else {
fmt.Fprintf(&b, "%s, last hour: %d now", hourName(key), h.Now[key])
}
}
if p, at := h.peak(key); at >= 0 && p > 0 {
fmt.Fprintf(&b, ", peak %d", p)
if s := h.slotAt(at); s != "" {
fmt.Fprintf(&b, " at %s UTC", s)
}
} else {
b.WriteString(", none in the hour")
}
switch len(h.Deploys) {
case 0:
b.WriteString(", no deploy in the hour")
case 1:
fmt.Fprintf(&b, ", deploy at %s UTC", h.Deploys[0].At)
default:
ats := make([]string, len(h.Deploys))
for i, d := range h.Deploys {
ats[i] = d.At
}
fmt.Fprintf(&b, ", %d deploys at %s UTC", len(h.Deploys), strings.Join(ats, ", "))
}
if g := h.gaps(); len(g) > 0 {
down := 0
for _, r := range g {
down += r[1] - r[0]
}
fmt.Fprintf(&b, ", no data for %d %s while the forge was down", down, pluralWord(down, "minute", "minutes"))
}
return b.String()
}
func pluralWord(n int, one, many string) string {
if n == 1 {
return one
}
return many
}
func f1(f float64) string { return strconv.FormatFloat(f, 'f', 1, 64) }
// slotX is the middle of slot i in drawing units.
func (h *opsHour) slotX(i int) float64 { return (float64(i) + 0.5) * hourW / float64(h.N) }
// limitLabelAt is where the limit's label starts, as a percentage of the
// plot: the first of a few places along the line that no deploy line
// crosses, so the word is never struck through by one.
func (h *opsHour) limitLabelAt() float64 {
const width = 24.0
for _, at := range []float64{1, 76, 40, 20, 60} {
clear := true
for _, d := range h.Deploys {
if p := d.Pct(h.N); p > at-1 && p < at+width {
clear = false
}
}
if clear {
return at
}
}
return 1
}
// hourChart is one measure's chart. tile draws its own time axis and
// deploy labels; the shared panel draws one of each for all four rows.
func hourChart(h *opsHour, key string, tile bool) template.HTML {
if h == nil || h.N < 2 {
return ""
}
top := h.top(key)
series := h.Series[key]
bars := hourBars(key)
slot := hourW / float64(h.N)
peak, _ := h.peak(key)
var b strings.Builder
kind := "line"
if bars {
kind = "bars"
}
fmt.Fprintf(&b, `<div class="ops-ch" data-hook="ops-chart" data-measure="%s" data-kind="%s" data-top="%s" data-peak="%d" data-deploys="%d"`,
key, kind, hourNum(top), peak, len(h.Deploys))
if h.Now[key] >= 0 {
fmt.Fprintf(&b, ` data-now="%d"`, h.Now[key])
}
if key == "errors" && h.NoErrors {
b.WriteString(` data-unavailable="true"`)
}
if key == "busy" && h.Limit > 0 {
fmt.Fprintf(&b, ` data-limit="%d"`, h.Limit)
}
b.WriteString(">\n")
// The scale: zero, the middle and the top. A middle that is not a
// whole number is left off a count, where half a request is nonsense.
b.WriteString(`<div class="ops-ch-y" aria-hidden="true">`)
fmt.Fprintf(&b, `<span style="top:0">%s</span>`, hourNum(top))
mid := top / 2
if !bars || mid == math.Trunc(mid) {
fmt.Fprintf(&b, `<span style="top:50%%">%s</span>`, hourNum(mid))
}
b.WriteString(`<span style="top:100%">0</span></div>`)
b.WriteString(`<div class="ops-ch-p">`)
fmt.Fprintf(&b, `<svg class="ops-ch-svg" viewBox="0 0 %g %g" preserveAspectRatio="none" role="img" aria-label="%s">`,
hourW, hourH, template.HTMLEscapeString(h.label(key)))
fmt.Fprintf(&b, `<path class="ops-ch-g" d="M0 0H%gM0 %gH%g"/><path class="ops-ch-z" data-hook="ops-zero" d="M0 %gH%g"/>`,
hourW, hourH/2, hourW, hourH, hourW)
for _, g := range h.gaps() {
fmt.Fprintf(&b, `<rect class="ops-ch-gap" data-hook="ops-gap" data-from="%d" data-to="%d" x="%s" y="0" width="%s" height="%g"/>`,
g[0], g[1], f1(float64(g[0])*slot), f1(float64(g[1]-g[0])*slot), hourH)
}
if key == "busy" && h.Limit > 0 {
fmt.Fprintf(&b, `<path class="ops-ch-lim" data-hook="ops-limit" data-limit="%d" d="M0 %sH%g"/>`, h.Limit, f1(h.y(float64(h.Limit), top)), hourW)
}
if tile {
for _, d := range h.Deploys {
fmt.Fprintf(&b, `<path class="ops-ch-dep" data-hook="ops-deploy" data-minute="%d" d="M%s 0V%g"/>`, d.Minute, f1(d.Pos*slot), hourH)
}
}
if bars {
cls := "warn"
if key == "errors" {
cls = "bad"
}
for i, v := range series {
if v <= 0 || !h.Up[i] || (key == "errors" && h.NoErrors) {
continue
}
y := math.Min(h.y(float64(v), top), hourH-3)
fmt.Fprintf(&b, `<rect class="ops-ch-bar %s" data-hook="ops-bar" data-minute="%d" data-value="%d" x="%s" y="%s" width="%s" height="%s"/>`,
cls, i, v, f1(float64(i)*slot+slot*0.12), f1(y), f1(slot*0.76), f1(hourH-y))
}
} else {
h.line(&b, key, series, top)
}
b.WriteString(`</svg>`)
if key == "busy" && h.Limit > 0 {
fmt.Fprintf(&b, `<span class="ops-ch-lt" data-hook="ops-limit-label" style="left:%s%%;top:%s%%">limit %d</span>`, f1(h.limitLabelAt()), f1(h.y(float64(h.Limit), top)), h.Limit)
}
if tile {
for _, l := range h.deployLabels() {
fmt.Fprintf(&b, `<span class="ops-ch-dl%s" data-hook="ops-deploy-label" style="left:%s%%" title="%s">%s</span>`, l.Side, f1(l.Pct), l.Tip, l.Text)
}
}
b.WriteString(`</div>`)
if tile {
b.WriteString(hourAxis(h, false))
}
b.WriteString("</div>")
return template.HTML(b.String())
}
// line draws a rate: a 2px line and a soft fill under each run of
// minutes a process was running in, and a dot on the last value. A run
// of one minute is a dot alone; a gap is a break.
func (h *opsHour) line(b *strings.Builder, key string, series []int, top float64) {
last := h.last(key)
var lastX, lastY float64
haveDot := false
for i := 0; i <= last; i++ {
if !h.Up[i] {
continue
}
j := i
for j <= last && h.Up[j] {
j++
}
var pts strings.Builder
for k := i; k < j; k++ {
op := "L"
if k == i {
op = "M"
}
fmt.Fprintf(&pts, "%s%s %s", op, f1(h.slotX(k)), f1(h.y(float64(series[k]), top)))
}
x0, x1 := h.slotX(i), h.slotX(j-1)
if j-i > 1 {
fmt.Fprintf(b, `<path class="ops-ch-fill" d="%sL%s %gL%s %gZ"/>`, pts.String(), f1(x1), hourH, f1(x0), hourH)
}
line := pts.String()
if j-i == 1 {
line += "L" + f1(x0) + " " + f1(h.y(float64(series[i]), top))
}
fmt.Fprintf(b, `<path class="ops-ch-line" data-hook="ops-line" d="%s"/>`, line)
lastX, lastY, haveDot = h.slotX(j-1), h.y(float64(series[j-1]), top), j-1 == last
i = j
}
if haveDot {
fmt.Fprintf(b, `<path class="ops-ch-dot" data-hook="ops-dot" d="M%s %sL%s %s"/>`, f1(lastX), f1(lastY), f1(lastX), f1(lastY))
}
}
// hourAxis is the clock under a chart: the ends and the middle on a
// tile, every fifteen minutes on the shared panel.
func hourAxis(h *opsHour, five bool) string {
if h.First.IsZero() {
return ""
}
var b strings.Builder
b.WriteString(`<div class="ops-ch-x" data-hook="ops-axis" aria-hidden="true">`)
for k, i := range h.ticks(five) {
cls := ""
if five && k%2 == 1 {
cls = ` class="opt"`
}
fmt.Fprintf(&b, `<span%s style="left:%s%%">%s</span>`, cls, f1((float64(i)+0.5)/float64(h.N)*100), h.slotAt(i))
}
b.WriteString(`</div>`)
return b.String()
}
// HourAxis is the panel's shared time axis.
func (h *opsHour) HourAxis() template.HTML { return template.HTML(hourAxis(h, true)) }
// deployLabel is the word beside a deploy line. Restarts too close
// together for their words to fit share the first one's label, which
// then counts them, rather than the later ones going unnamed.
type deployLabel struct {
Pct float64
Count int
Text, Tip string
Side string // " r" when the word sits to the left of its line
}
func (h *opsHour) deployLabels() []deployLabel {
var out []deployLabel
var ats [][]string
prev := -100.0
for _, d := range h.Deploys {
p := d.Pct(h.N)
if len(out) > 0 && p-prev < 14 {
ats[len(ats)-1] = append(ats[len(ats)-1], d.At)
continue
}
prev = p
out = append(out, deployLabel{Pct: p})
ats = append(ats, []string{d.At})
}
for i := range out {
out[i].Count = len(ats[i])
out[i].Text, out[i].Tip = "deploy", "Restarted at "+ats[i][0]+" UTC"
if n := len(ats[i]); n > 1 {
out[i].Text = fmt.Sprintf("%d deploys", n)
out[i].Tip = "Restarted at " + strings.Join(ats[i], ", ") + " UTC"
}
if out[i].Pct > 78 {
out[i].Side = " r"
}
}
return out
}
// DeployBands are the restarts as one mark through every row.
func (h *opsHour) DeployBands() template.HTML {
var b strings.Builder
for _, d := range h.Deploys {
fmt.Fprintf(&b, `<i class="ops-hr-dep" data-hook="ops-hour-deploy" data-minute="%d" style="left:%s%%"></i>`, d.Minute, f1(d.Pct(h.N)))
}
for _, l := range h.deployLabels() {
fmt.Fprintf(&b, `<span class="ops-hr-dl%s" style="left:%s%%" title="%s">%s</span>`, l.Side, f1(l.Pct), l.Tip, l.Text)
}
return template.HTML(b.String())
}
// Hover is what the panel reads a minute from, already worded: this file
// formats every number so the script never does.
func (h *opsHour) Hover() template.HTMLAttr {
type row struct {
K string `json:"k"`
Name string `json:"name"`
V []string `json:"v"`
}
out := struct {
At []string `json:"at"`
Rows []row `json:"rows"`
}{At: make([]string, h.N)}
for i := range out.At {
out.At[i] = h.slotAt(i)
if out.At[i] != "" {
out.At[i] += " UTC"
}
}
for _, k := range hourKeys {
r := row{K: k, Name: hourShort(k), V: make([]string, h.N)}
for i := range r.V {
if h.Up[i] && !(k == "errors" && h.NoErrors) {
r.V[i] = strconv.Itoa(h.Series[k][i])
} else {
r.V[i] = "no data"
}
}
out.Rows = append(out.Rows, r)
}
b, _ := json.Marshal(out)
return template.HTMLAttr(`data-hover="` + template.HTMLEscapeString(string(b)) + `"`)
}
// Rows are the measures in strip order, for the panel's template.
func (h *opsHour) Rows() []string { return hourKeys }
// hourParam reads ?hour=: absent or 0 is the tiles, 1 the panel, and a
// measure's name the panel with that row picked out.
func hourParam(q url.Values) (open bool, focus string) {
switch v := q.Get("hour"); v {
case "", "0":
return false, ""
case "renders", "slow", "errors", "busy":
return true, v
}
return true, ""
}
// KeptParams are the request's other query parameters, carried through
// the form's reload so a sorted table stays sorted.
func (v opsView) KeptParams() [][2]string {
var out [][2]string
for k, vs := range v.Query {
if k == "hour" {
continue
}
for _, x := range vs {
out = append(out, [2]string{k, x})
}
}
sort.Slice(out, func(i, j int) bool { return out[i][0]+"="+out[i][1] < out[j][0]+"="+out[j][1] })
return out
}
// HourView names for the page's wrapper.
func (v opsView) ViewName() string {
if v.HourOpen {
return "hour"
}
return "tiles"
}
// hourRowView is a measure's left-hand block on the shared panel.
type hourRowView struct {
Key, Name, Value, Small, Hint, Tone string
Now int
}
func hourRow(v opsView, key string) hourRowView {
r := hourRowView{Key: key, Name: hourName(key), Hint: v.HourHint(key)}
switch key {
case "renders":
hv := v.HourView()
r.Now, r.Value = hv.Now["renders"], hv.RendersFigure()
case "slow":
r.Now = v.RoutesOver
r.Value, r.Small = strconv.Itoa(r.Now), fmt.Sprintf("of %d", v.Routes)
if r.Now > 0 {
r.Tone = "ov-warn"
}
case "errors":
hv := v.HourView()
r.Now, r.Value, r.Small = hv.Now["errors"], hv.ErrorsFigure(), "in the hour"
if r.Now > 0 {
r.Tone = "ov-bad"
}
default:
r.Now = v.LiveSubprocess
r.Value = strconv.Itoa(r.Now)
r.Small = "peak " + strconv.Itoa(v.PeakSubprocess)
if b := v.StoreBound(); b > 0 {
r.Small = "of " + strconv.Itoa(b) + " ยท " + r.Small
}
}
return r
}