//go:build !js && !wasip1
package obsduck
import (
"context"
"database/sql"
"errors"
"os"
"path/filepath"
"sync"
"sync/atomic"
"testing"
"time"
"craftdesign.group/quarry/internal/obs"
)
func reopenSink(t *testing.T, batch int) (*Sink, string) {
t.Helper()
path := filepath.Join(t.TempDir(), "ops.duckdb")
s, err := Open(path, batch)
if err != nil {
t.Skipf("duckdb unavailable: %v", err)
}
t.Cleanup(func() { s.Close() })
return s, path
}
func tableRows(t *testing.T, s *Sink, table string) int {
t.Helper()
var n int
if err := s.db.QueryRow(`SELECT count(*) FROM ` + table).Scan(&n); err != nil {
t.Fatal(err)
}
return n
}
// Rows written before a reopen are read back after it, rows still
// buffered across it are not lost, and every reader works on the new handle.
func TestReopenKeepsLandedAndBufferedRows(t *testing.T) {
s, _ := reopenSink(t, 1024)
now := time.Now()
s.Render(obs.Render{At: now, Path: "/a", Elapsed: 5 * time.Millisecond, Status: 200})
s.Hit(obs.Hit{At: now, Door: obs.DoorWeb, URL: "/r/q", Repo: "q", UA: "curl/8", Status: 200})
s.Flush()
s.Render(obs.Render{At: now, Path: "/b", Elapsed: 7 * time.Millisecond, Status: 200})
if err := s.Reopen(); err != nil {
t.Fatalf("FAIL: reopen: %v", err)
}
if n := tableRows(t, s, "render"); n != 1 {
t.Errorf("FAIL: %d renders landed after reopen, want 1", n)
}
tr, err := s.Trend("", 2)
if err != nil || len(tr) != 1 || tr[0].N != 2 {
t.Errorf("FAIL: trend after reopen = %v, %v; want one day of 2 (buffered row included)", tr, err)
}
h, err := s.History(obs.HistoryQuery{})
if err != nil || len(h) != 2 {
t.Errorf("FAIL: history after reopen = %d rows, %v; want 2", len(h), err)
}
if _, err := s.Traffic(obs.TrafficQuery{Now: now}); err != nil {
t.Errorf("FAIL: traffic after reopen: %v", err)
}
if _, err := s.Hourly(obs.HourlyQuery{}); err != nil {
t.Errorf("FAIL: hourly after reopen: %v", err)
}
if _, err := s.Swarm(now, 7); err != nil {
t.Errorf("FAIL: swarm after reopen: %v", err)
}
if !t.Failed() {
t.Log("ok")
}
}
// Writers, readers and reopens together: no error but cancellation, no
// deadlock, and every row accounted for.
func TestReopenAmongWritersAndReaders(t *testing.T) {
s, _ := reopenSink(t, 16)
now := time.Now()
stop := make(chan struct{})
var wg sync.WaitGroup
var written atomic.Int64
var bad atomic.Value
note := func(err error) {
if err != nil && !errors.Is(err, context.Canceled) {
bad.Store(err)
}
}
for w := 0; w < 3; w++ {
wg.Add(1)
go func() {
defer wg.Done()
for {
select {
case <-stop:
return
default:
}
s.Render(obs.Render{At: now, Path: "/p", Status: 200})
s.Hit(obs.Hit{At: now, Door: obs.DoorWeb, URL: "/r/q", Status: 200, UA: "curl/8"})
s.Subprocess(obs.Subprocess{At: now, Verb: "log"})
written.Add(1)
s.Flush()
}
}()
}
readers := []func() error{
func() error { _, err := s.Traffic(obs.TrafficQuery{Now: now}); return err },
func() error { _, err := s.History(obs.HistoryQuery{}); return err },
func() error { _, err := s.Hourly(obs.HourlyQuery{}); return err },
func() error { _, err := s.Swarm(now, 7); return err },
func() error { _, err := s.Prune(24 * time.Hour); return err },
}
for _, r := range readers {
wg.Add(1)
go func() {
defer wg.Done()
for {
select {
case <-stop:
return
default:
}
note(r())
}
}()
}
wg.Add(1)
go func() {
defer wg.Done()
for {
select {
case <-stop:
return
default:
}
note(s.Reopen())
time.Sleep(20 * time.Millisecond)
}
}()
time.Sleep(1500 * time.Millisecond)
close(stop)
done := make(chan struct{})
go func() { wg.Wait(); close(done) }()
select {
case <-done:
case <-time.After(30 * time.Second):
t.Fatal("FAIL: deadlock: workers did not finish")
}
if e := bad.Load(); e != nil {
t.Errorf("FAIL: unexpected error: %v", e)
}
s.Flush()
got := tableRows(t, s, "render") + tableRows(t, s, "hit") + tableRows(t, s, "subprocess")
want := 3*int(written.Load()) - int(s.Dropped())
if got != want || got == 0 {
t.Errorf("FAIL: %d rows stored, want %d (written %d of each, dropped %d)", got, want, written.Load(), s.Dropped())
}
if n, _ := s.Reopens(); n == 0 {
t.Errorf("FAIL: no reopen happened")
}
if !t.Failed() {
t.Log("ok")
}
}
// Reopen after Close does nothing but report it; Close during a reopen
// completes.
func TestReopenAndCloseAreSafeAgainstEachOther(t *testing.T) {
s, _ := reopenSink(t, 8)
if err := s.Close(); err != nil {
t.Fatal(err)
}
if err := s.Reopen(); !errors.Is(err, errClosed) {
t.Errorf("FAIL: reopen after close = %v, want errClosed", err)
}
if n, _ := s.Reopens(); n != 0 {
t.Errorf("FAIL: a reopen after close was counted")
}
if _, err := s.History(obs.HistoryQuery{}); err == nil {
t.Errorf("FAIL: a read after close succeeded")
}
s2, _ := reopenSink(t, 8)
var wg sync.WaitGroup
for i := 0; i < 4; i++ {
wg.Add(1)
go func() {
defer wg.Done()
for j := 0; j < 20; j++ {
s2.Reopen()
}
}()
}
time.Sleep(30 * time.Millisecond)
closed := make(chan error, 1)
go func() { closed <- s2.Close() }()
select {
case err := <-closed:
if err != nil {
t.Errorf("FAIL: close during reopen: %v", err)
}
case <-time.After(20 * time.Second):
t.Fatal("FAIL: close during reopen did not complete")
}
wg.Wait()
if !t.Failed() {
t.Log("ok")
}
}
// A reopen that cannot open the file leaves the sink reporting why, keeps
// buffering, reads fail without a panic, and a later reopen recovers.
func TestFailedReopenIsReportedAndRecovers(t *testing.T) {
s, path := reopenSink(t, 1024)
s.Render(obs.Render{At: time.Now(), Path: "/a", Status: 200})
s.Hit(obs.Hit{At: time.Now(), Door: obs.DoorWeb, URL: "/r/q", Status: 200})
s.Flush()
aside := path + ".aside"
if err := os.Rename(path, aside); err != nil {
t.Fatal(err)
}
if err := os.Mkdir(path, 0o755); err != nil {
t.Fatal(err)
}
if err := s.Reopen(); err == nil {
t.Fatalf("FAIL: reopen of an unopenable path succeeded")
}
// The hit's hour is still open; a rollup flush with no handle must
// skip quietly, not file the outage as a write failure of its own.
s.Hit(obs.Hit{At: time.Now().Add(-2 * time.Hour), Door: obs.DoorWeb, URL: "/r/q", Status: 200})
s.flushRollup(false)
if s.Err() == nil {
t.Errorf("FAIL: Err does not report the failed reopen")
}
s.Render(obs.Render{At: time.Now(), Path: "/b", Status: 200})
s.Flush()
if _, err := s.History(obs.HistoryQuery{}); err == nil {
t.Errorf("FAIL: a read without a handle succeeded")
}
if n, _ := s.Reopens(); n != 0 {
t.Errorf("FAIL: a failed reopen was counted")
}
if err := os.Remove(path); err != nil {
t.Fatal(err)
}
if err := os.Rename(aside, path); err != nil {
t.Fatal(err)
}
if err := s.Reopen(); err != nil {
t.Fatalf("FAIL: reopen did not recover: %v", err)
}
if s.Err() != nil {
t.Errorf("FAIL: Err still set after recovery: %v", s.Err())
}
s.Flush()
if n := tableRows(t, s, "render"); n != 2 {
t.Errorf("FAIL: %d renders after recovery, want 2 (the row buffered meanwhile included)", n)
}
if !t.Failed() {
t.Log("ok")
}
}
// The health a reopen reports moves with it.
func TestReopenMovesTheHealthFields(t *testing.T) {
s, _ := reopenSink(t, 8)
r := obs.NewRecorder(8, s)
if h := r.Health(); h.Reopens != 0 || !h.LastReopen.IsZero() {
t.Errorf("FAIL: health before a reopen = %d, %v", h.Reopens, h.LastReopen)
}
before := time.Now()
if err := s.Reopen(); err != nil {
t.Fatal(err)
}
h := r.Health()
if h.Reopens != 1 || h.LastReopen.Before(before) {
t.Errorf("FAIL: health after a reopen = %d, %v", h.Reopens, h.LastReopen)
}
if !t.Failed() {
t.Log("ok")
}
}
// A write that finds the handle gone puts its rows back, capped at max,
// and nothing is lost but what the cap counts.
func TestRowsGoBackWhenTheHandleGoesBeforeTheWrite(t *testing.T) {
s, _ := reopenSink(t, 4)
real := s.write
var down atomic.Bool
down.Store(true)
s.write = func(ctx context.Context, r []obs.Render, ss []obs.Subprocess, h []obs.Hit) error {
if down.Load() {
return errUnavailable
}
return real(ctx, r, ss, h)
}
const total = 100
for i := 0; i < total; i++ {
s.Render(obs.Render{At: time.Now(), Path: "/a", Status: 200})
s.Flush()
}
down.Store(false)
s.Flush()
got := tableRows(t, s, "render")
if got == 0 || got > s.max || got+int(s.Dropped()) != total {
t.Errorf("FAIL: %d stored, %d dropped of %d written (max %d)", got, s.Dropped(), total, s.max)
}
if s.Err() != nil {
t.Errorf("FAIL: an unavailable write was filed as a failure: %v", s.Err())
}
if !t.Failed() {
t.Log("ok")
}
}
// Close that arrives while a reopen is stuck opening returns within its
// budget, and counts the rows it had nowhere to write.
func TestCloseDuringAStuckReopenIsBounded(t *testing.T) {
s, _ := reopenSink(t, 1024)
s.closeBudget = 300 * time.Millisecond
entered := make(chan struct{})
s.open = func(ctx context.Context, _ string) (*sql.DB, error) {
close(entered)
<-ctx.Done()
return nil, ctx.Err()
}
reopened := make(chan error, 1)
go func() { reopened <- s.Reopen() }()
<-entered
for i := 0; i < 5; i++ {
s.Render(obs.Render{At: time.Now(), Path: "/a", Status: 200})
}
start := time.Now()
closed := make(chan error, 1)
go func() { closed <- s.Close() }()
select {
case <-closed:
case <-time.After(5 * time.Second):
t.Fatal("FAIL: Close did not return")
}
if d := time.Since(start); d > s.closeBudget+time.Second {
t.Errorf("FAIL: Close took %v, budget %v", d, s.closeBudget)
}
if err := <-reopened; err == nil {
t.Errorf("FAIL: the stuck reopen reported success")
}
if s.Dropped() != 5 {
t.Errorf("FAIL: dropped %d, want the 5 rows Close could not write", s.Dropped())
}
if !t.Failed() {
t.Log("ok")
}
}
// A reader holding a connection makes Reopen wait; once it lets go both
// succeed, and a read that queued behind the waiting reopen succeeds too.
func TestReopenWaitsForAHeldConnection(t *testing.T) {
s, _ := reopenSink(t, 8)
s.Hit(obs.Hit{At: time.Now(), Door: obs.DoorWeb, URL: "/r/q", Status: 200, UA: "curl/8"})
s.Flush()
if err := s.rlockDB(); err != nil {
t.Fatal(err)
}
conn, err := s.db.Conn(context.Background())
if err != nil {
t.Fatal(err)
}
var n int
if err := conn.QueryRowContext(context.Background(), `SELECT count(*) FROM hit`).Scan(&n); err != nil || n != 1 {
t.Fatalf("FAIL: held connection read %d, %v", n, err)
}
reopened := make(chan error, 1)
go func() { reopened <- s.Reopen() }()
read := make(chan error, 1)
time.Sleep(100 * time.Millisecond)
go func() { _, err := s.Traffic(obs.TrafficQuery{Now: time.Now()}); read <- err }()
select {
case <-reopened:
t.Fatal("FAIL: reopen finished while a connection was held")
case <-time.After(300 * time.Millisecond):
}
conn.Close()
s.dbMu.RUnlock()
for _, c := range []chan error{reopened, read} {
select {
case err := <-c:
if err != nil {
t.Errorf("FAIL: %v", err)
}
case <-time.After(20 * time.Second):
t.Fatal("FAIL: still waiting after the connection was released")
}
}
if !t.Failed() {
t.Log("ok")
}
}