package index
import (
"bytes"
"context"
"errors"
"fmt"
"log"
"math/rand"
"os"
"os/exec"
"path/filepath"
"strings"
"syscall"
"testing"
"time"
)
// A rebuild suspends the search triggers and folds rows that never reach
// search_fts; a process that ends anywhere in that window must leave an
// index that opens clean (#403). These tests end the process in every
// way a restart can: a return with the work undone, and a SIGKILL.
const bulkChildEnv = "QUARRY_BULKKILL_DIR"
func bulkRecords(cycle, batch, n int) []CheckinRecord {
recs := make([]CheckinRecord, n)
for i := range recs {
recs[i] = CheckinRecord{
Hash: fmt.Sprintf("%016x%048x", cycle*1000+batch, i),
Date: "2026-09-15T00:00:00",
User: "dan",
Comment: fmt.Sprintf("cycle %d batch %d row %d zebra quokka", cycle, batch, i),
}
}
return recs
}
// mustBeClean asserts what a restart needs: the file is sound, the
// startup search check reports no disagreement, and one live fold of an
// already-held row works.
func mustBeClean(t *testing.T, path, label string) {
t.Helper()
ix, err := Open(path)
if err != nil {
t.Fatalf("%s: open: %v", label, err)
}
defer ix.Close()
var res string
if err := ix.db.QueryRow(`PRAGMA integrity_check`).Scan(&res); err != nil || res != "ok" {
t.Fatalf("%s: integrity_check = %q, %v", label, res, err)
}
var logged bytes.Buffer
log.SetOutput(&logged)
defer log.SetOutput(os.Stderr)
ix.VerifySearch()
if strings.Contains(logged.String(), "disagrees") {
t.Fatalf("%s: startup search check failed: %s", label, logged.String())
}
if err := ix.FoldCheckins(bulkRecords(0, 0, 3)); err != nil {
t.Fatalf("%s: live fold: %v", label, err)
}
if err := ix.FoldCheckins(bulkRecords(0, 0, 3)); err != nil {
t.Fatalf("%s: live refold: %v", label, err)
}
}
// The process returns with the triggers dropped and rows folded.
func TestInterruptedBulkRebuildOpensClean(t *testing.T) {
for _, stage := range []string{"folding", "resumed", "rebuilt"} {
t.Run(stage, func(t *testing.T) {
path := filepath.Join(t.TempDir(), "index.db")
ix, err := Open(path)
if err != nil {
t.Fatal(err)
}
if err := ix.Reset(); err != nil {
t.Fatal(err)
}
if err := ix.SuspendDerivedMaintenance(); err != nil {
t.Fatal(err)
}
if err := ix.FoldCheckins(bulkRecords(1, 0, 50)); err != nil {
t.Fatal(err)
}
if stage != "folding" {
if err := ix.ResumeDerivedMaintenance(); err != nil {
t.Fatal(err)
}
}
if stage == "rebuilt" {
if err := ix.RebuildSearch(); err != nil {
t.Fatal(err)
}
}
ix.Close() // stands in for the process ending
mustBeClean(t, path, stage)
})
}
}
// TestBulkKillChild is the other process: it rebuilds in a loop until
// it is killed or its own deadline passes.
func TestBulkKillChild(t *testing.T) {
dir := os.Getenv(bulkChildEnv)
if dir == "" {
t.Skip("runs only as the child of TestSigkillDuringBulkRebuild")
}
time.AfterFunc(60*time.Second, func() { os.Exit(3) })
ix, err := Open(filepath.Join(dir, "index.db"))
if err != nil {
t.Fatal(err)
}
fmt.Println("ready")
for cycle := 1; ; cycle++ {
if err := ix.Reset(); err != nil {
t.Fatal(err)
}
if err := ix.SuspendDerivedMaintenance(); err != nil {
t.Fatal(err)
}
for b := 0; b < 20; b++ {
if err := ix.FoldCheckins(bulkRecords(cycle, b, 400)); err != nil {
t.Fatal(err)
}
}
if err := ix.ResumeDerivedMaintenance(); err != nil {
t.Fatal(err)
}
if err := ix.RebuildSearch(); err != nil {
t.Fatal(err)
}
}
}
func TestSigkillDuringBulkRebuild(t *testing.T) {
if os.Getenv(bulkChildEnv) != "" {
t.Skip("child")
}
if testing.Short() {
t.Skip("spawns processes")
}
dir := t.TempDir()
path := filepath.Join(dir, "index.db")
rng := rand.New(rand.NewSource(403))
const kills = 12
for i := 0; i < kills; i++ {
ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
cmd := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestBulkKillChild$", "-test.count=1")
cmd.Env = append(os.Environ(), bulkChildEnv+"="+dir)
pr, pw, err := os.Pipe()
if err != nil {
cancel()
t.Fatal(err)
}
cmd.Stdout = pw
cmd.Stderr = pw
if err := cmd.Start(); err != nil {
cancel()
t.Fatal(err)
}
pw.Close()
ready := make(chan struct{})
go func() {
defer pr.Close()
buf := make([]byte, 4096)
var seen string
closed := false
for {
n, err := pr.Read(buf)
seen += string(buf[:n])
if !closed && strings.Contains(seen, "ready") {
closed = true
close(ready)
}
if err != nil {
return
}
}
}()
// Past the child's open, then a random point in its rebuild.
select {
case <-ready:
case <-ctx.Done():
cancel()
t.Fatalf("kill %d: the child never opened", i)
}
time.Sleep(time.Duration(20+rng.Intn(1500)) * time.Millisecond)
_ = cmd.Process.Kill()
_ = cmd.Wait()
cancel()
mustBeClean(t, path, fmt.Sprintf("kill %d", i))
}
}
const resetChildEnv = "QUARRY_RESETKILL_DIR"
// TestResetKillChild is the other process of
// TestSigkillBetweenResetDrops: it kills itself right after Reset drops
// search_doc, before search_fts and meta go.
func TestResetKillChild(t *testing.T) {
dir := os.Getenv(resetChildEnv)
if dir == "" {
t.Skip("runs only as the child of TestSigkillBetweenResetDrops")
}
ix, err := Open(filepath.Join(dir, "index.db"))
if err != nil {
t.Fatal(err)
}
resetDropped = func(table string) {
if table == "search_doc" {
_ = syscall.Kill(os.Getpid(), syscall.SIGKILL)
select {}
}
}
_ = ix.Reset()
os.Exit(4) // the kill never came
}
// A reset of an index whose search agrees with its content, ended
// between its drops, must leave an index that still agrees (#403).
func TestSigkillBetweenResetDrops(t *testing.T) {
if os.Getenv(resetChildEnv) != "" {
t.Skip("child")
}
if testing.Short() {
t.Skip("spawns processes")
}
dir := t.TempDir()
path := filepath.Join(dir, "index.db")
ix, err := Open(path)
if err != nil {
t.Fatal(err)
}
if err := ix.FoldCheckins(bulkRecords(1, 0, 50)); err != nil {
t.Fatal(err)
}
ix.Close()
cmd := exec.Command(os.Args[0], "-test.run=^TestResetKillChild$", "-test.count=1")
cmd.Env = append(os.Environ(), resetChildEnv+"="+dir)
out, err := cmd.CombinedOutput()
var ee *exec.ExitError
if !errors.As(err, &ee) || ee.ProcessState.Sys().(syscall.WaitStatus).Signal() != syscall.SIGKILL {
t.Fatalf("FAIL: the child was not killed inside Reset: %v\n%s", err, out)
}
mustBeClean(t, path, "reset kill")
if !t.Failed() {
t.Log("ok")
}
}