Skip to content
quarry Artifact [ff0c7d5f2d]
Login

Artifact [ff0c7d5f2d]

Artifact ff0c7d5f2d73a1c3a7846917ad49695e8aea40127339ea80f431bb0c43c47567:


//go:build !js && !wasip1

package obsduck

import (
	"context"
	"database/sql"
	"errors"
	"os"
	"path/filepath"
	"sync"
	"sync/atomic"
	"testing"
	"time"

	"craftdesign.group/quarry/internal/obs"
)

func reopenSink(t *testing.T, batch int) (*Sink, string) {
	t.Helper()
	path := filepath.Join(t.TempDir(), "ops.duckdb")
	s, err := Open(path, batch)
	if err != nil {
		t.Skipf("duckdb unavailable: %v", err)
	}
	t.Cleanup(func() { s.Close() })
	return s, path
}

func tableRows(t *testing.T, s *Sink, table string) int {
	t.Helper()
	var n int
	if err := s.db.QueryRow(`SELECT count(*) FROM ` + table).Scan(&n); err != nil {
		t.Fatal(err)
	}
	return n
}

// Rows written before a reopen are read back after it, rows still
// buffered across it are not lost, and every reader works on the new handle.
func TestReopenKeepsLandedAndBufferedRows(t *testing.T) {
	s, _ := reopenSink(t, 1024)
	now := time.Now()
	s.Render(obs.Render{At: now, Path: "/a", Elapsed: 5 * time.Millisecond, Status: 200})
	s.Hit(obs.Hit{At: now, Door: obs.DoorWeb, URL: "/r/q", Repo: "q", UA: "curl/8", Status: 200})
	s.Flush()
	s.Render(obs.Render{At: now, Path: "/b", Elapsed: 7 * time.Millisecond, Status: 200})
	if err := s.Reopen(); err != nil {
		t.Fatalf("FAIL: reopen: %v", err)
	}
	if n := tableRows(t, s, "render"); n != 1 {
		t.Errorf("FAIL: %d renders landed after reopen, want 1", n)
	}
	tr, err := s.Trend("", 2)
	if err != nil || len(tr) != 1 || tr[0].N != 2 {
		t.Errorf("FAIL: trend after reopen = %v, %v; want one day of 2 (buffered row included)", tr, err)
	}
	h, err := s.History(obs.HistoryQuery{})
	if err != nil || len(h) != 2 {
		t.Errorf("FAIL: history after reopen = %d rows, %v; want 2", len(h), err)
	}
	if _, err := s.Traffic(obs.TrafficQuery{Now: now}); err != nil {
		t.Errorf("FAIL: traffic after reopen: %v", err)
	}
	if _, err := s.Hourly(obs.HourlyQuery{}); err != nil {
		t.Errorf("FAIL: hourly after reopen: %v", err)
	}
	if _, err := s.Swarm(now, 7); err != nil {
		t.Errorf("FAIL: swarm after reopen: %v", err)
	}
	if !t.Failed() {
		t.Log("ok")
	}
}

// Writers, readers and reopens together: no error but cancellation, no
// deadlock, and every row accounted for.
func TestReopenAmongWritersAndReaders(t *testing.T) {
	s, _ := reopenSink(t, 16)
	now := time.Now()
	stop := make(chan struct{})
	var wg sync.WaitGroup
	var written atomic.Int64
	var bad atomic.Value
	note := func(err error) {
		if err != nil && !errors.Is(err, context.Canceled) {
			bad.Store(err)
		}
	}
	for w := 0; w < 3; w++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			for {
				select {
				case <-stop:
					return
				default:
				}
				s.Render(obs.Render{At: now, Path: "/p", Status: 200})
				s.Hit(obs.Hit{At: now, Door: obs.DoorWeb, URL: "/r/q", Status: 200, UA: "curl/8"})
				s.Subprocess(obs.Subprocess{At: now, Verb: "log"})
				written.Add(1)
				s.Flush()
			}
		}()
	}
	readers := []func() error{
		func() error { _, err := s.Traffic(obs.TrafficQuery{Now: now}); return err },
		func() error { _, err := s.History(obs.HistoryQuery{}); return err },
		func() error { _, err := s.Hourly(obs.HourlyQuery{}); return err },
		func() error { _, err := s.Swarm(now, 7); return err },
		func() error { _, err := s.Prune(24 * time.Hour); return err },
	}
	for _, r := range readers {
		wg.Add(1)
		go func() {
			defer wg.Done()
			for {
				select {
				case <-stop:
					return
				default:
				}
				note(r())
			}
		}()
	}
	wg.Add(1)
	go func() {
		defer wg.Done()
		for {
			select {
			case <-stop:
				return
			default:
			}
			note(s.Reopen())
			time.Sleep(20 * time.Millisecond)
		}
	}()
	time.Sleep(1500 * time.Millisecond)
	close(stop)
	done := make(chan struct{})
	go func() { wg.Wait(); close(done) }()
	select {
	case <-done:
	case <-time.After(30 * time.Second):
		t.Fatal("FAIL: deadlock: workers did not finish")
	}
	if e := bad.Load(); e != nil {
		t.Errorf("FAIL: unexpected error: %v", e)
	}
	s.Flush()
	got := tableRows(t, s, "render") + tableRows(t, s, "hit") + tableRows(t, s, "subprocess")
	want := 3*int(written.Load()) - int(s.Dropped())
	if got != want || got == 0 {
		t.Errorf("FAIL: %d rows stored, want %d (written %d of each, dropped %d)", got, want, written.Load(), s.Dropped())
	}
	if n, _ := s.Reopens(); n == 0 {
		t.Errorf("FAIL: no reopen happened")
	}
	if !t.Failed() {
		t.Log("ok")
	}
}

// Reopen after Close does nothing but report it; Close during a reopen
// completes.
func TestReopenAndCloseAreSafeAgainstEachOther(t *testing.T) {
	s, _ := reopenSink(t, 8)
	if err := s.Close(); err != nil {
		t.Fatal(err)
	}
	if err := s.Reopen(); !errors.Is(err, errClosed) {
		t.Errorf("FAIL: reopen after close = %v, want errClosed", err)
	}
	if n, _ := s.Reopens(); n != 0 {
		t.Errorf("FAIL: a reopen after close was counted")
	}
	if _, err := s.History(obs.HistoryQuery{}); err == nil {
		t.Errorf("FAIL: a read after close succeeded")
	}

	s2, _ := reopenSink(t, 8)
	var wg sync.WaitGroup
	for i := 0; i < 4; i++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			for j := 0; j < 20; j++ {
				s2.Reopen()
			}
		}()
	}
	time.Sleep(30 * time.Millisecond)
	closed := make(chan error, 1)
	go func() { closed <- s2.Close() }()
	select {
	case err := <-closed:
		if err != nil {
			t.Errorf("FAIL: close during reopen: %v", err)
		}
	case <-time.After(20 * time.Second):
		t.Fatal("FAIL: close during reopen did not complete")
	}
	wg.Wait()
	if !t.Failed() {
		t.Log("ok")
	}
}

// A reopen that cannot open the file leaves the sink reporting why, keeps
// buffering, reads fail without a panic, and a later reopen recovers.
func TestFailedReopenIsReportedAndRecovers(t *testing.T) {
	s, path := reopenSink(t, 1024)
	s.Render(obs.Render{At: time.Now(), Path: "/a", Status: 200})
	s.Hit(obs.Hit{At: time.Now(), Door: obs.DoorWeb, URL: "/r/q", Status: 200})
	s.Flush()
	aside := path + ".aside"
	if err := os.Rename(path, aside); err != nil {
		t.Fatal(err)
	}
	if err := os.Mkdir(path, 0o755); err != nil {
		t.Fatal(err)
	}
	if err := s.Reopen(); err == nil {
		t.Fatalf("FAIL: reopen of an unopenable path succeeded")
	}
	// The hit's hour is still open; a rollup flush with no handle must
	// skip quietly, not file the outage as a write failure of its own.
	s.Hit(obs.Hit{At: time.Now().Add(-2 * time.Hour), Door: obs.DoorWeb, URL: "/r/q", Status: 200})
	s.flushRollup(false)
	if s.Err() == nil {
		t.Errorf("FAIL: Err does not report the failed reopen")
	}
	s.Render(obs.Render{At: time.Now(), Path: "/b", Status: 200})
	s.Flush()
	if _, err := s.History(obs.HistoryQuery{}); err == nil {
		t.Errorf("FAIL: a read without a handle succeeded")
	}
	if n, _ := s.Reopens(); n != 0 {
		t.Errorf("FAIL: a failed reopen was counted")
	}
	if err := os.Remove(path); err != nil {
		t.Fatal(err)
	}
	if err := os.Rename(aside, path); err != nil {
		t.Fatal(err)
	}
	if err := s.Reopen(); err != nil {
		t.Fatalf("FAIL: reopen did not recover: %v", err)
	}
	if s.Err() != nil {
		t.Errorf("FAIL: Err still set after recovery: %v", s.Err())
	}
	s.Flush()
	if n := tableRows(t, s, "render"); n != 2 {
		t.Errorf("FAIL: %d renders after recovery, want 2 (the row buffered meanwhile included)", n)
	}
	if !t.Failed() {
		t.Log("ok")
	}
}

// The health a reopen reports moves with it.
func TestReopenMovesTheHealthFields(t *testing.T) {
	s, _ := reopenSink(t, 8)
	r := obs.NewRecorder(8, s)
	if h := r.Health(); h.Reopens != 0 || !h.LastReopen.IsZero() {
		t.Errorf("FAIL: health before a reopen = %d, %v", h.Reopens, h.LastReopen)
	}
	before := time.Now()
	if err := s.Reopen(); err != nil {
		t.Fatal(err)
	}
	h := r.Health()
	if h.Reopens != 1 || h.LastReopen.Before(before) {
		t.Errorf("FAIL: health after a reopen = %d, %v", h.Reopens, h.LastReopen)
	}
	if !t.Failed() {
		t.Log("ok")
	}
}

// A write that finds the handle gone puts its rows back, capped at max,
// and nothing is lost but what the cap counts.
func TestRowsGoBackWhenTheHandleGoesBeforeTheWrite(t *testing.T) {
	s, _ := reopenSink(t, 4)
	real := s.write
	var down atomic.Bool
	down.Store(true)
	s.write = func(ctx context.Context, r []obs.Render, ss []obs.Subprocess, h []obs.Hit) error {
		if down.Load() {
			return errUnavailable
		}
		return real(ctx, r, ss, h)
	}
	const total = 100
	for i := 0; i < total; i++ {
		s.Render(obs.Render{At: time.Now(), Path: "/a", Status: 200})
		s.Flush()
	}
	down.Store(false)
	s.Flush()
	got := tableRows(t, s, "render")
	if got == 0 || got > s.max || got+int(s.Dropped()) != total {
		t.Errorf("FAIL: %d stored, %d dropped of %d written (max %d)", got, s.Dropped(), total, s.max)
	}
	if s.Err() != nil {
		t.Errorf("FAIL: an unavailable write was filed as a failure: %v", s.Err())
	}
	if !t.Failed() {
		t.Log("ok")
	}
}

// Close that arrives while a reopen is stuck opening returns within its
// budget, and counts the rows it had nowhere to write.
func TestCloseDuringAStuckReopenIsBounded(t *testing.T) {
	s, _ := reopenSink(t, 1024)
	s.closeBudget = 300 * time.Millisecond
	entered := make(chan struct{})
	s.open = func(ctx context.Context, _ string) (*sql.DB, error) {
		close(entered)
		<-ctx.Done()
		return nil, ctx.Err()
	}
	reopened := make(chan error, 1)
	go func() { reopened <- s.Reopen() }()
	<-entered
	for i := 0; i < 5; i++ {
		s.Render(obs.Render{At: time.Now(), Path: "/a", Status: 200})
	}
	start := time.Now()
	closed := make(chan error, 1)
	go func() { closed <- s.Close() }()
	select {
	case <-closed:
	case <-time.After(5 * time.Second):
		t.Fatal("FAIL: Close did not return")
	}
	if d := time.Since(start); d > s.closeBudget+time.Second {
		t.Errorf("FAIL: Close took %v, budget %v", d, s.closeBudget)
	}
	if err := <-reopened; err == nil {
		t.Errorf("FAIL: the stuck reopen reported success")
	}
	if s.Dropped() != 5 {
		t.Errorf("FAIL: dropped %d, want the 5 rows Close could not write", s.Dropped())
	}
	if !t.Failed() {
		t.Log("ok")
	}
}

// A reader holding a connection makes Reopen wait; once it lets go both
// succeed, and a read that queued behind the waiting reopen succeeds too.
func TestReopenWaitsForAHeldConnection(t *testing.T) {
	s, _ := reopenSink(t, 8)
	s.Hit(obs.Hit{At: time.Now(), Door: obs.DoorWeb, URL: "/r/q", Status: 200, UA: "curl/8"})
	s.Flush()
	if err := s.rlockDB(); err != nil {
		t.Fatal(err)
	}
	conn, err := s.db.Conn(context.Background())
	if err != nil {
		t.Fatal(err)
	}
	var n int
	if err := conn.QueryRowContext(context.Background(), `SELECT count(*) FROM hit`).Scan(&n); err != nil || n != 1 {
		t.Fatalf("FAIL: held connection read %d, %v", n, err)
	}
	reopened := make(chan error, 1)
	go func() { reopened <- s.Reopen() }()
	read := make(chan error, 1)
	time.Sleep(100 * time.Millisecond)
	go func() { _, err := s.Traffic(obs.TrafficQuery{Now: time.Now()}); read <- err }()
	select {
	case <-reopened:
		t.Fatal("FAIL: reopen finished while a connection was held")
	case <-time.After(300 * time.Millisecond):
	}
	conn.Close()
	s.dbMu.RUnlock()
	for _, c := range []chan error{reopened, read} {
		select {
		case err := <-c:
			if err != nil {
				t.Errorf("FAIL: %v", err)
			}
		case <-time.After(20 * time.Second):
			t.Fatal("FAIL: still waiting after the connection was released")
		}
	}
	if !t.Failed() {
		t.Log("ok")
	}
}