package web
import (
"io"
"net/http"
"net/http/httptest"
"regexp"
"strings"
"testing"
)
// robotsAllows reads a robots.txt as the large crawlers do: the longest
// matching rule decides, Allow wins a tie, and * matches any run of
// characters.
func robotsAllows(rules, path string) bool {
best, allow := -1, true
for _, line := range strings.Split(rules, "\n") {
verb, pat, ok := strings.Cut(line, ": ")
if !ok || (verb != "Allow" && verb != "Disallow") {
continue
}
re := regexp.MustCompile("^" + strings.ReplaceAll(regexp.QuoteMeta(pat), `\*`, ".*"))
if !re.MatchString(path) || len(pat) < best {
continue
}
if len(pat) > best || verb == "Allow" {
best, allow = len(pat), verb == "Allow"
}
}
return allow
}
// robots.txt turns crawlers away from the history they walked at full
// cost (a tarball, a patch and a file view at every check-in) and keeps
// the default branch's pages open to them.
func TestRobotsKeepsCrawlersOffEveryCheckinsCopy(t *testing.T) {
srv := httptest.NewServer(HandlerWith(viewRegistry(t, nil), nil, nil, nil))
defer srv.Close()
resp, err := http.Get(srv.URL + "/robots.txt")
if err != nil {
t.Fatal(err)
}
body, _ := io.ReadAll(resp.Body)
resp.Body.Close()
if resp.StatusCode != 200 || !strings.HasPrefix(resp.Header.Get("Content-Type"), "text/plain") {
t.Fatalf("FAIL: /robots.txt answered %d %q", resp.StatusCode, resp.Header.Get("Content-Type"))
}
hash := strings.Repeat("ab", 32)
for path, want := range map[string]bool{
"/r/dmestas/fossil/archive/" + hash: false,
"/r/dmestas/fossil/checkin/" + hash + "/patch": false,
"/r/dmestas/fossil/checkin/" + hash + "/diff": false,
"/r/dmestas/fossil/file/" + hash + "/extsrc/sqlite3.c": false,
"/r/dmestas/fossil/tree/" + hash + "/src": false,
"/r/dmestas/fossil/tree/trunk/src": true,
"/r/dmestas/fossil/file/trunk/src/main.c": true,
"/r/dmestas/fossil": true,
"/r/dmestas/fossil/timeline": true,
"/r/dmestas/fossil/checkin/" + hash: true,
} {
if got := robotsAllows(string(body), path); got != want {
t.Errorf("FAIL: %s allowed=%v, want %v", path, got, want)
}
}
t.Log("ok")
}