//go:build !js && !wasip1
package obsduck
import (
"path/filepath"
"testing"
"time"
"craftdesign.group/quarry/internal/obs"
)
const chrome = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/153.0.0.0 Safari/537.36"
// Before a day-old address is blanked, a distributed crawler's request
// keeps its range in the swarm table; a reader who read on, a page the
// swarm is not counted on, a signed-in reader and a self-named crawler do not.
func TestTheSwarmKeepsOnlyOneRequestBrowsersAskingForHashes(t *testing.T) {
// A batch larger than the rows, so none is written in the background
// while the prune runs.
s, err := Open(filepath.Join(t.TempDir(), "ops.duckdb"), 64)
if err != nil {
t.Fatal(err)
}
defer s.Close()
old := time.Now().Add(-AddrKept - time.Hour)
diff := "/r/tcl/checkin/0123456789abcdef0123/diff"
for _, h := range []obs.Hit{
// the swarm: one request each, by hash
{At: old, Door: obs.DoorWeb, URL: diff, Repo: "tcl", Addr: "203.0.113.9", UA: chrome, Visitor: 1, Hints: "ae", HeadSet: "h1"},
{At: old, Door: obs.DoorWeb, URL: "/r/tk/0123456789abcdef0123/diff", Repo: "tk", Addr: "203.0.113.200", UA: chrome, Visitor: 2, Hints: "ae", HeadSet: "h1"},
// a reader who went on to a second page
{At: old, Door: obs.DoorWeb, URL: diff, Repo: "tcl", Addr: "198.51.100.7", UA: chrome, Visitor: 3},
{At: old.Add(time.Minute), Door: obs.DoorWeb, URL: "/r/tcl/timeline", Repo: "tcl", Addr: "198.51.100.7", UA: chrome, Visitor: 3},
// a page the swarm is not counted on
{At: old, Door: obs.DoorWeb, URL: "/r/tcl/wiki/Home", Repo: "tcl", Addr: "198.51.100.8", UA: chrome, Visitor: 4},
// signed in
{At: old, Door: obs.DoorWeb, URL: diff, Repo: "tcl", Addr: "198.51.100.9", UA: chrome, Visitor: 5, Account: "dan"},
// a crawler that names itself
{At: old, Door: obs.DoorWeb, URL: diff, Repo: "tcl", Addr: "198.51.100.10", UA: "Mozilla/5.0 (compatible; Googlebot/2.1)", Visitor: 6},
// too fresh: its day is not over
{At: time.Now(), Door: obs.DoorWeb, URL: diff, Repo: "tcl", Addr: "203.0.113.50", UA: chrome, Visitor: 7},
} {
s.Hit(h)
}
if _, err := s.Prune(0); err != nil {
t.Fatal(err)
}
sw, err := s.Swarm(time.Now(), 7)
if err != nil {
t.Fatal(err)
}
if sw.Requests != 2 {
t.Fatalf("FAIL: the swarm kept %d requests, want 2: %+v", sw.Requests, sw)
}
if len(sw.Prefixes) != 1 || sw.Prefixes[0].Prefix != "203.0.113.0/24" || sw.Prefixes[0].Requests != 2 {
t.Errorf("FAIL: the ranges: %+v", sw.Prefixes)
}
if len(sw.Shapes) != 2 || len(sw.HeadSets) != 1 || sw.HeadSets[0].Count != 2 {
t.Errorf("FAIL: shapes %+v, header sets %+v", sw.Shapes, sw.HeadSets)
}
var addrs int
if err := s.db.QueryRow(`SELECT count(*) FROM hit WHERE addr IS NOT NULL OR prefix IS NOT NULL AND ts < ?`, time.Now().Add(-AddrKept)).Scan(&addrs); err != nil {
t.Fatal(err)
}
if addrs != 1 { // only the fresh row keeps its address
t.Errorf("FAIL: %d rows keep an address or range", addrs)
}
// A second prune finds nothing new: the swept rows have no address.
if _, err := s.Prune(0); err != nil {
t.Fatal(err)
}
if sw, _ = s.Swarm(time.Now(), 7); sw.Requests != 2 {
t.Errorf("FAIL: a second sweep kept rows again: %d", sw.Requests)
}
}