package obs
import "testing"
func TestClassify(t *testing.T) {
for _, tc := range []struct {
name string
f Facts
kind string
crawler string
}{
{"a named crawler", Facts{UA: "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"}, KindCrawler, "Googlebot"},
{"an AI crawler", Facts{UA: "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.2"}, KindCrawler, "GPTBot"},
{"a crawler that signed in is still a crawler", Facts{UA: "ClaudeBot/1.0", Account: "x"}, KindCrawler, "ClaudeBot"},
{"a verified account", Facts{UA: "curl/8.4.0", Account: "dmestas"}, KindPerson, ""},
{"an anonymous sync", Facts{UA: "zit/0.6.6", Door: DoorSync, Client: "zit"}, KindUnsure, ""},
{"no user agent", Facts{}, KindAutomated, ""},
{"a library", Facts{UA: "python-requests/2.31"}, KindAutomated, ""},
{"an unnamed bot", Facts{UA: "Mozilla/5.0 (compatible; SomeCrawler/1.0)"}, KindAutomated, ""},
{"a browser", Facts{UA: "Mozilla/5.0 (X11; Linux x86_64; rv:131.0) Gecko/20100101 Firefox/131.0"}, KindPerson, ""},
{"something else", Facts{UA: "Lynx/2.9"}, KindUnsure, ""},
} {
got := Classify(tc.f)
if got.Kind != tc.kind || got.Crawler != tc.crawler {
t.Errorf("FAIL: %s: %+v, want kind %s crawler %q", tc.name, got, tc.kind, tc.crawler)
}
}
}
// The client is named the way a reader would name it, and a sync round
// that already named its client is believed over its user agent.
func TestTheClientIsNamed(t *testing.T) {
for ua, want := range map[string]string{
"zit/0.6.6": "zit",
"zig/0.15": "zit",
"Fossil/2.26": "fossil",
"curl/8.4.0": "curl",
"Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 Chrome/130.0 Safari/537.36 Edg/130.0": "Edge",
"Mozilla/5.0 (Macintosh) AppleWebKit/605.1.15 Version/18.0 Safari/605.1.15": "Safari",
} {
if got := Classify(Facts{UA: ua}).Client; got != want {
t.Errorf("FAIL: %q named %q, want %q", ua, got, want)
}
}
if got := Classify(Facts{UA: "Mozilla/5.0", Door: DoorSync, Client: "fossil"}).Client; got != "fossil" {
t.Errorf("FAIL: the sync round's own client was overridden: %q", got)
}
}
// A crawler is checked against the network it must come from. Its name
// sent from anywhere else is an impostor, and an impostor is a script;
// a crawler with no rule is only unverified, never accused.
func TestCrawlersAreCheckedAgainstTheirNetwork(t *testing.T) {
ua := "Mozilla/5.0 (compatible; Googlebot/2.1)"
if r := Classify(Facts{UA: ua, ASN: 15169}); !r.Verified || r.Kind != KindCrawler {
t.Errorf("FAIL: Googlebot from Google: %+v", r)
}
if r := Classify(Facts{UA: ua, ASN: 45102}); !r.Impostor || r.Kind != KindAutomated || r.Crawler != "Googlebot" {
t.Errorf("FAIL: Googlebot from Alibaba Cloud: %+v", r)
}
if r := Classify(Facts{UA: ua}); r.Verified || r.Impostor || r.Kind != KindCrawler {
t.Errorf("FAIL: Googlebot from an unknown network was judged: %+v", r)
}
if r := Classify(Facts{UA: "ClaudeBot/1.0", ASN: 24940}); r.Impostor {
t.Errorf("FAIL: a crawler with no rule was called an impostor: %+v", r)
}
}
// A browser on a hosting network is not taken for a person.
func TestABrowserOnARentedServerIsUnsure(t *testing.T) {
ua := "Mozilla/5.0 (X11; Linux x86_64; rv:131.0) Gecko/20100101 Firefox/131.0"
if r := Classify(Facts{UA: ua, ASN: 24940}); r.Kind != KindUnsure {
t.Errorf("FAIL: Firefox from Hetzner reads %s", r.Kind)
}
if r := Classify(Facts{UA: ua, ASN: 7922}); r.Kind != KindPerson {
t.Errorf("FAIL: Firefox from Comcast reads %s", r.Kind)
}
}