package index
import (
"strings"
"testing"
"time"
)
// bigDoc is a generated listing: one line per hit, far past
// bigSnippetBytes, the shape of a pkgsrc PLIST.
func bigDoc(lines int) string {
var b strings.Builder
b.WriteString("header line without it\n")
for i := 0; i < lines; i++ {
b.WriteString("lib/python3.12/site-packages/widget/mod")
b.WriteString(strings.Repeat("x", i%7))
b.WriteString(".py\n")
}
return b.String()
}
// A document too big to snippet still gets one, cut from its first
// matching line with the token marked (quarry#449).
func TestSearchBigDocumentGetsALineSnippet(t *testing.T) {
ix := openIndex(t)
body := bigDoc(8000)
if len(body) <= bigSnippetBytes {
t.Fatalf("fixture is %d bytes, under the threshold", len(body))
}
if err := ix.ReplaceTree("v1", []TreeText{
{Path: "big/PLIST", Body: body},
{Path: "small.txt", Body: "a widget in a short file"},
}); err != nil {
t.Fatal(err)
}
rs, err := ix.Search("WIDGET", "", 10)
if err != nil {
t.Fatal(err)
}
got := map[string]string{}
for _, r := range rs {
got[r.Ref[strings.LastIndex(r.Ref, "/")+1:]] = r.Snippet
}
if s := got["PLIST"]; !strings.Contains(s, "⟦widget⟧") || strings.Contains(s, "header line") || len(s) > 400 {
t.Errorf("FAIL: big snippet = %q", s)
}
small, err := ix.Search("widget short", "", 10)
if err != nil {
t.Fatal(err)
}
var want string
if err := ix.db.QueryRow(`SELECT snippet(search_fts, 0, '⟦', '⟧', '…', 12) FROM search_fts
WHERE search_fts MATCH '"widget" "short"' AND rowid = (SELECT id FROM search_doc WHERE ref LIKE '%small.txt')`).Scan(&want); err != nil {
t.Fatal(err)
}
if len(small) != 1 || small[0].Snippet != want {
t.Errorf("FAIL: small snippet %+v, want %q", small, want)
}
brief, err := ix.SearchBrief("WIDGET", "", 10)
if err != nil || len(brief) != len(rs) {
t.Fatalf("brief %d rows (%v), full %d", len(brief), err, len(rs))
}
for i := range rs {
if brief[i].Ref != rs[i].Ref || brief[i].Rank != rs[i].Rank {
t.Errorf("FAIL: row %d order differs from the snippet-free ranking", i)
}
}
if !t.Failed() {
t.Log("ok")
}
}
func TestLineSnippetCutsAroundTheToken(t *testing.T) {
long := strings.Repeat("a ", 200) + "needle" + strings.Repeat(" b", 200)
s := lineSnippet("first\n"+long+"\nlast", []string{"needle"})
if !strings.HasPrefix(s, "…") || !strings.HasSuffix(s, "…") || !strings.Contains(s, "⟦needle⟧") || len(s) > 300 {
t.Errorf("FAIL: cut = %q", s)
}
if s := lineSnippet("xneedle needlex\nplain needle.", []string{"needle"}); s != "plain ⟦needle⟧." {
t.Errorf("FAIL: whole-token match = %q", s)
}
if s := lineSnippet("say \u201cneedle\u201d now", []string{"needle"}); s != "say \u201c⟦needle⟧\u201d now" {
t.Errorf("FAIL: curly quotes are boundaries: %q", s)
}
for _, doc := range []string{"CAF\u00c9", "caf\u00e9", "cafe", "cafe\u0301"} {
for _, q := range []string{"CAF\u00c9", "caf\u00e9", "cafe"} {
if s := lineSnippet("a "+doc+" b", []string{q}); !strings.Contains(s, "⟦"+doc+"⟧") {
t.Errorf("FAIL: %q does not match %q: %q", q, doc, s)
}
}
}
if s := lineSnippet("\n \nfirst real line\nsecond", []string{"zzz"}); s != "first real line" {
t.Errorf("FAIL: fallback = %q", s)
}
if !t.Failed() {
t.Log("ok")
}
}
// Prints the time of the page's query on a document of thousands of
// hits; the number is for the ticket, the check is that it ends.
func TestSearchBigDocumentTiming(t *testing.T) {
ix := openIndex(t)
var tree []TreeText
for i := 0; i < 5; i++ {
tree = append(tree, TreeText{Path: "lang/python" + string(rune('a'+i)) + "/PLIST", Body: bigDoc(7000 + i)})
}
if err := ix.ReplaceTree("v1", tree); err != nil {
t.Fatal(err)
}
start := time.Now()
rs, err := ix.Search("py", "", 10)
if err != nil || len(rs) != 5 {
t.Fatalf("rows %d, err %v", len(rs), err)
}
t.Logf("search over 5 docs of ~7000 hits: %s", time.Since(start))
}
// The page's snippet query reads each kept row's text by key. Joining the
// search_all view instead (a UNION ALL, which SQLite cannot push an id
// into) built every text in the index to keep fifty rows: on pkgsrc that
// ran the connection past its 256 MB ceiling and the page failed.
func TestSnippetQueryReadsTextByKey(t *testing.T) {
ix := openIndex(t)
rows, err := ix.db.Query(`EXPLAIN QUERY PLAN `+searchRanked+snippetSelect,
`"x"`, "", "", 50, bigSnippetBytes, bigSnippetBytes, `"x"`)
if err != nil {
t.Fatal(err)
}
defer rows.Close()
plan := ""
for rows.Next() {
var a, b, c int
var detail string
if err := rows.Scan(&a, &b, &c, &detail); err != nil {
t.Fatal(err)
}
plan += detail + "\n"
}
for _, bad := range []string{"SCAN checkin", "SCAN search_doc", "COMPOUND", "search_all"} {
if strings.Contains(plan, bad) {
t.Errorf("FAIL: snippet query plan holds %q:\n%s", bad, plan)
}
}
if !t.Failed() {
t.Log("ok")
}
}