package web
import (
"fmt"
"html/template"
"time"
"craftdesign.group/quarry/internal/obs"
)
// fleetSwarmView is /fleet/swarm: the distributed crawler the traffic
// page counts as people, as the sweep has kept it (obs.Swarm).
type fleetSwarmView struct {
Kept bool // the forge keeps request history at all
Days int
Ranges []int // the windows offered
Requests string
Prefixes int
Networks int
Daily []trBar
Busiest []trBar // the hours with the most requests
Table []swarmRange
Nets []trBar
Hints []trBar
HeadSets []trBar
Langs []trBar
Shapes []trBar
Repos []trBar
Agents []trBar
Places []trBar
}
type swarmRange struct {
Prefix, Network, Country, Requests, Seen string
Days int64
}
func swarmView(sw *obs.Swarm, days int) fleetSwarmView {
if days <= 0 || days > 90 {
days = 7
}
v := fleetSwarmView{Days: days, Ranges: []int{1, 7, 30, 90}}
if sw == nil {
return v
}
v.Kept = true
v.Requests = thousands(int(sw.Requests))
v.Prefixes, v.Networks = len(sw.Prefixes), len(sw.Networks)
series := func(ds []obs.SwarmDay, layout string) []trBar {
var top int64
for _, d := range ds {
top = max(top, d.Requests)
}
out := make([]trBar, 0, len(ds))
for _, d := range ds {
out = append(out, trBar{Label: d.At.UTC().Format(layout), Value: shortCount(d.Requests),
Note: fmt.Sprintf("%d ranges · %d networks", d.Prefixes, d.Networks), Pct: pct(d.Requests, top)})
}
return out
}
v.Daily = series(sw.Days, "Mon 2 Jan")
hours := append([]obs.SwarmDay(nil), sw.Hours...)
for i := 1; i < len(hours); i++ { // the busiest first; a few dozen rows
for j := i; j > 0 && hours[j].Requests > hours[j-1].Requests; j-- {
hours[j], hours[j-1] = hours[j-1], hours[j]
}
}
if len(hours) > 12 {
hours = hours[:12]
}
v.Busiest = series(hours, "Mon 2 Jan 15:00 UTC")
for _, p := range sw.Prefixes {
v.Table = append(v.Table, swarmRange{Prefix: p.Prefix, Network: p.Network, Country: p.Country,
Requests: thousands(int(p.Requests)), Days: p.Days, Seen: seenSpan(p.First, p.Last)})
}
var top int64
for _, n := range sw.Networks {
top = max(top, n.Requests)
}
for _, n := range sw.Networks {
v.Nets = append(v.Nets, trBar{Label: n.Network, Note: fmt.Sprintf("AS%d · %d ranges", n.ASN, n.Prefixes),
Value: shortCount(n.Requests), Pct: pct(n.Requests, top)})
}
counts := func(cs []obs.SwarmCount, mono bool, empty string) []trBar {
var top int64
for _, c := range cs {
top = max(top, c.Count)
}
out := make([]trBar, 0, len(cs))
for _, c := range cs {
label := c.Key
if label == "" {
label = empty
}
out = append(out, trBar{Label: label, Value: shortCount(c.Count), Pct: pct(c.Count, top), Mono: mono})
}
return out
}
v.Hints = counts(sw.Hints, true, "none of a browser's headers")
v.HeadSets = counts(sw.HeadSets, true, "none")
v.Langs = counts(sw.Langs, false, "no language")
v.Shapes = counts(sw.Shapes, true, "")
v.Repos = counts(sw.Repos, false, "no repository")
v.Agents = counts(sw.Agents, true, "no user agent")
v.Places = counts(sw.Countries, false, "unknown")
return v
}
func pct(n, top int64) int {
if top <= 0 {
return 0
}
return int(n * 100 / top)
}
func seenSpan(first, last time.Time) string {
f, l := first.UTC().Format("2 Jan"), last.UTC().Format("2 Jan")
if f == l {
return f
}
return f + " – " + l
}
var fleetSwarmPage = page(template.Must(template.Must(base.Clone()).Parse(trBarsTmpl + `{{define "content"}}{{with .FleetSwarm}}
{{template "tr-views" "swarm"}}
<section class="tr-card" data-hook="swarm-page">
<div class="tr-h"><h1>Distributed crawlers</h1></div>
<p class="tr-hint">A crawler that sends each request from a different address under a browser's name: one request per address, hundreds of networks in a minute, almost all for pages named by a hash. Each request alone reads as a person, so the traffic page counts them as people. Kept {{if .Kept}}90 days{{end}} by network range (a /24 or a /48, never an address), with how each request presented itself; a day's rows arrive the next day, when its addresses are blanked.</p>
<nav class="tabs" aria-label="window">{{$d := .Days}}{{range .Ranges}}<a href="/fleet/swarm?days={{.}}"{{if eq . $d}} aria-current="page" class="on"{{end}}>{{.}} day{{if ne . 1}}s{{end}}</a> {{end}}</nav>
{{if not .Kept}}<p class="tr-none" data-hook="swarm-none">This forge keeps no request history.</p>{{else if not .Table}}<p class="tr-none" data-hook="swarm-empty">Nothing kept in this window yet. The first rows arrive a day after this page was deployed.</p>{{else}}
<p data-hook="swarm-total"><b>{{.Requests}}</b> requests from <b>{{.Prefixes}}</b> ranges in <b>{{.Networks}}</b> networks.</p>{{end}}
</section>
{{if .Table}}<div class="tr-grid">
<section class="tr-card"><div class="tr-h"><h2>By day</h2><span>requests, ranges and networks</span></div>{{template "tr-bars" .Daily}}</section>
<section class="tr-card"><div class="tr-h"><h2>Busiest hours</h2><span>where the bursts are</span></div>{{template "tr-bars" .Busiest}}</section>
</div>
<section class="tr-card" data-hook="swarm-ranges"><div class="tr-h"><h2>Ranges</h2><span>most requests first · the list to handle</span></div>
<div class="tw"><table class="tr-t"><thead><tr><th>range</th><th>network</th><th>country</th><th class="n">requests</th><th class="n">days</th><th>seen</th></tr></thead><tbody>
{{range .Table}}<tr data-hook="swarm-range"><td class="tr-mono">{{.Prefix}}</td><td>{{.Network}}</td><td>{{.Country}}</td><td class="n">{{.Requests}}</td><td class="n">{{.Days}}</td><td>{{.Seen}}</td></tr>{{end}}
</tbody></table></div><p class="tr-hint">The full list, as JSON: <code>GET /api/v1/fleet/swarm?days={{.Days}}</code>.</p></section>
<div class="tr-grid">
<section class="tr-card"><div class="tr-h"><h2>Networks</h2><span>who owns the ranges</span></div>{{template "tr-bars" .Nets}}</section>
<section class="tr-card"><div class="tr-h"><h2>Countries</h2><span>the edge's guess</span></div>{{template "tr-bars" .Places}}</section>
<section class="tr-card"><div class="tr-h"><h2>Browser headers sent</h2><span>a page load sends ch fs fm=navigate fd al ae</span></div>{{template "tr-bars" .Hints}}</section>
<section class="tr-card"><div class="tr-h"><h2>Header sets</h2><span>a digest of the header names: one tool, one set</span></div>{{template "tr-bars" .HeadSets}}</section>
<section class="tr-card"><div class="tr-h"><h2>Languages</h2><span>Accept-Language's first</span></div>{{template "tr-bars" .Langs}}</section>
<section class="tr-card"><div class="tr-h"><h2>What they ask for</h2><span>pages, hashes taken out</span></div>{{template "tr-bars" .Shapes}}</section>
<section class="tr-card"><div class="tr-h"><h2>Repositories</h2></div>{{template "tr-bars" .Repos}}</section>
<section class="tr-card"><div class="tr-h"><h2>User agents</h2><span>what they claim to be</span></div>{{template "tr-bars" .Agents}}</section>
</div>{{end}}
{{end}}{{end}}`)))