package obs
import "strings"
// The kinds of reader the traffic page tells apart. A closed set: the
// page draws one band per kind.
const (
KindPerson = "person"
KindCrawler = "crawler" // a search or AI crawler that names itself
KindAutomated = "automated" // a script, a library, a crawler that names nothing
KindUnsure = "unsure"
)
// Reader is what the forge concludes about who sent a request.
type Reader struct {
Kind string
// Client is the software, as a reader would name it: "zit",
// "fossil", "Firefox", "curl".
Client string
// Crawler is the crawler a request names itself as, "" for none. It
// is a CLAIM: anyone can send Googlebot's user agent, and only the
// address it came from can say whether it is Google.
Crawler string
// Verified is a crawler whose network is the one it must come from;
// Impostor is one whose network is known and is not. Neither is set
// for a crawler the forge has no rule for.
Verified, Impostor bool
}
// Facts is what Classify reads: the parts of a Hit that bear on who
// sent it.
type Facts struct {
UA, Account, Door, Client string
// ASN is the network the request came from, 0 when unknown.
ASN uint32
}
// Classify decides who a request came from.
//
// Ordered by how much each signal is worth. A crawler that names itself
// is believed to be a crawler (whether it is THAT crawler is the
// network's question, not this one's). A verified account is a person.
// A user agent that is a library, or that is missing, is a script. A
// sync client is a person or a mirror and nothing in one request says
// which, so it is unsure until its pattern over time does.
func Classify(f Facts) Reader {
ua := f.UA
low := strings.ToLower(ua)
if name := crawlerOf(low); name != "" {
r := Reader{Kind: KindCrawler, Client: name, Crawler: name}
if want, ok := verifiedBy[name]; ok && f.ASN != 0 {
for _, asn := range want {
if asn == f.ASN {
r.Verified = true
}
}
// Somebody sending a crawler's name from a network that is
// not the crawler's is a script, and a script that lies.
if !r.Verified {
r.Kind, r.Impostor = KindAutomated, true
}
}
return r
}
client := clientOf(ua, low, f.Client)
switch {
case f.Account != "":
return Reader{Kind: KindPerson, Client: client}
case f.Door == DoorSync:
return Reader{Kind: KindUnsure, Client: client}
case ua == "":
return Reader{Kind: KindAutomated, Client: "no user agent"}
case isLibrary(low) || selfDeclaredBot(low):
return Reader{Kind: KindAutomated, Client: client}
case isBrowser(ua) && networks[f.ASN].hosting:
// A browser on a rented server: a person behind a VPN, or a
// script wearing a browser's name. One request cannot say.
return Reader{Kind: KindUnsure, Client: client}
case isBrowser(ua):
return Reader{Kind: KindPerson, Client: client}
}
return Reader{Kind: KindUnsure, Client: client}
}
// crawlers are the crawlers that name themselves, by the token their
// user agent carries. Ordered so a more specific token wins: Google's
// other agents before Googlebot.
var crawlers = []struct{ token, name string }{
{"googlebot", "Googlebot"},
{"google-inspectiontool", "Googlebot"},
{"googleother", "GoogleOther"},
{"google-extended", "Google-Extended"},
{"bingbot", "bingbot"},
{"applebot", "Applebot"},
{"gptbot", "GPTBot"},
{"oai-searchbot", "OAI-SearchBot"},
{"chatgpt-user", "ChatGPT-User"},
{"claudebot", "ClaudeBot"},
{"claude-user", "Claude-User"},
{"claude-searchbot", "Claude-SearchBot"},
{"perplexitybot", "PerplexityBot"},
{"duckduckbot", "DuckDuckBot"},
{"yandexbot", "YandexBot"},
{"baiduspider", "Baiduspider"},
{"bytespider", "Bytespider"},
{"amazonbot", "Amazonbot"},
{"ccbot", "CCBot"},
{"meta-externalagent", "Meta"},
{"facebookexternalhit", "Meta"},
{"petalbot", "PetalBot"},
{"ahrefsbot", "AhrefsBot"},
{"semrushbot", "SemrushBot"},
{"mj12bot", "MJ12bot"},
{"dotbot", "DotBot"},
{"seznambot", "SeznamBot"},
{"qwantbot", "Qwantbot"},
{"mojeekbot", "MojeekBot"},
{"swh-", "Software Heritage"},
{"softwareheritage", "Software Heritage"},
}
func crawlerOf(low string) string {
for _, c := range crawlers {
if strings.Contains(low, c.token) {
return c.name
}
}
return ""
}
// libraries are user agents that are a program's HTTP library rather
// than anybody's browser.
var libraries = []string{
"curl/", "wget/", "python-requests", "python-urllib", "python-httpx", "aiohttp",
"go-http-client", "okhttp", "axios/", "node-fetch", "undici", "libwww-perl",
"java/", "apache-httpclient", "ruby", "php/", "guzzlehttp", "scrapy",
"headlesschrome", "phantomjs", "httpclient", "reqwest", "deno/", "bun/",
}
func isLibrary(low string) bool {
for _, l := range libraries {
if strings.Contains(low, l) {
return true
}
}
return false
}
// selfDeclaredBot is a crawler that says it is one without saying
// whose.
func selfDeclaredBot(low string) bool {
for _, w := range []string{"bot", "crawl", "spider", "slurp", "scan", "fetcher", "monitor", "preview"} {
if strings.Contains(low, w) {
return true
}
}
return false
}
func isBrowser(ua string) bool {
return strings.HasPrefix(ua, "Mozilla/5.0") &&
(strings.Contains(ua, "Firefox/") || strings.Contains(ua, "Chrome/") ||
strings.Contains(ua, "Safari/") || strings.Contains(ua, "Edg/"))
}
// clientOf names the software. A sync round already named its client;
// otherwise the user agent says, most specific token first.
func clientOf(ua, low, synced string) string {
if synced != "" && synced != "other" {
return synced
}
switch {
case strings.HasPrefix(ua, "zig/"), strings.Contains(ua, "zit/"):
return "zit"
case strings.HasPrefix(low, "fossil/"):
return "fossil"
case strings.Contains(ua, "Edg/"):
return "Edge"
case strings.Contains(ua, "Firefox/"):
return "Firefox"
case strings.Contains(ua, "Chrome/"):
return "Chrome"
case strings.Contains(ua, "Safari/"):
return "Safari"
}
if i := strings.IndexAny(ua, "/ "); i > 0 {
return ua[:i]
}
return ua
}