package api
import (
"sort"
"time"
"craftdesign.group/quarry/internal/children"
"craftdesign.group/quarry/internal/obs"
)
// Running now (#361): every live zit process the forge started, what
// started it, how long it has run against how long that kind of call
// usually takes, and its memory. The counter on the strip says how many
// are running; this says which, and why — which is the only form of
// that fact anyone can act on.
// opsRunning is one live child process.
type opsRunning struct {
Verb string `json:"verb"`
Repo string `json:"repo"`
// Cause is the request's address ("GET /r/fossil/timeline"), or the
// background activity (ingest, warm, sync front, store write…),
// empty when nobody said.
Cause string `json:"cause,omitempty"`
Started string `json:"started"`
AgeMs int64 `json:"ageMs"`
// UsualMs is the median duration of this verb over the calls still
// in the window, 0 when there are too few to say.
UsualMs int64 `json:"usualMs,omitempty"`
// RSS is the process's resident memory, 0 when it cannot be read
// (not Linux, or the process ended between listing and reading).
RSS uint64 `json:"rss,omitempty"`
// Slow marks a call past three times its usual duration.
Slow bool `json:"slow,omitempty"`
}
// usualSamples is how many finished calls of a verb it takes before
// their median is a usual duration rather than an anecdote.
const usualSamples = 3
// slowAfter is the age below which nothing is marked slow, however
// quick its usual: a 300 ms listing that usually takes 80 ms is not
// news.
const slowAfter = time.Second
// procMatchWithin is how far apart a child's registration and its
// process's start may be and still be the same child.
const procMatchWithin = 2 * time.Second
// opsRunningNow lists the live children with their usual durations and
// memory, oldest first.
func (d Deps) opsRunningNow(now time.Time, finished []obs.Subprocess) []opsRunning {
list := children.Default.Running
if d.Running != nil {
list = d.Running
}
kids := list()
if len(kids) == 0 {
return []opsRunning{}
}
usual := usualByVerb(finished)
procs := childProcs()
used := make([]bool, len(procs))
out := make([]opsRunning, 0, len(kids))
for _, k := range kids {
age := now.Sub(k.Start)
r := opsRunning{Verb: k.Verb, Repo: k.Repo, Cause: k.Cause,
Started: k.Start.UTC().Format(time.RFC3339), AgeMs: max(0, age.Milliseconds()),
UsualMs: usual[k.Verb].Milliseconds()}
if u := usual[k.Verb]; u > 0 && age >= slowAfter && age > 3*u {
r.Slow = true
}
// The process is this child's when its verb matches and it
// started nearest the registration; each process answers once.
best, bestGap := -1, procMatchWithin
for i, p := range procs {
if used[i] || p.Verb != k.Verb {
continue
}
gap := p.Start.Sub(k.Start)
if gap < 0 {
gap = -gap
}
if gap <= bestGap {
best, bestGap = i, gap
}
}
if best >= 0 {
used[best] = true
r.RSS = procs[best].RSS
}
out = append(out, r)
}
return out
}
// usualByVerb is each verb's median duration over the finished calls,
// for verbs with enough of them.
func usualByVerb(subs []obs.Subprocess) map[string]time.Duration {
by := map[string][]time.Duration{}
for _, s := range subs {
by[s.Verb] = append(by[s.Verb], s.Duration)
}
out := make(map[string]time.Duration, len(by))
for v, ds := range by {
if len(ds) < usualSamples {
continue
}
sort.Slice(ds, func(i, j int) bool { return ds[i] < ds[j] })
out[v] = ds[len(ds)/2]
}
return out
}
// childProc is one of this process's child processes as /proc shows it.
type childProc struct {
PID int
Verb string
Start time.Time
RSS uint64
}