Skip to content
quarry Artifact [35fa3e88a2]
Login

Artifact [35fa3e88a2]

Artifact 35fa3e88a2af86a022ac79c36d0fc7967f6f701f1a91335ccd082eb1cde14061:


//go:build !js && !wasip1

package obsduck

import (
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"strings"
	"testing"
	"time"

	"craftdesign.group/quarry/internal/obs"
)

func residentMB() string {
	out, err := exec.Command("ps", "-o", "rss=", "-p", strconv.Itoa(os.Getpid())).Output()
	if err != nil {
		return "?"
	}
	kb, _ := strconv.Atoi(strings.TrimSpace(string(out)))
	return fmt.Sprintf("%d MB", kb/1024)
}

// TestMemExperiment is an experiment, not a check: it documents the
// measurement behind ReopenEvery. DuckDB's native memory grows under
// repeated appends and wide reads and is given back only by closing the
// database. It is skipped unless MEMEXP is set, and only a Linux run
// counts, because macOS keeps the memory after a close.
//
//	MEMEXP=keep   go test ./internal/obsduck -run TestMemExperiment -count=1 -v -timeout 9m
//	MEMEXP=reopen go test ./internal/obsduck -run TestMemExperiment -count=1 -v -timeout 9m
//
// It loads 1.3M hits, then 400 times records 256 hits, flushes, and runs
// Traffic over 24h and History, printing resident memory every 50
// iterations; with reopen it also calls Reopen every 100 iterations.
func TestMemExperiment(t *testing.T) {
	mode := os.Getenv("MEMEXP")
	if mode == "" {
		t.Skip("set MEMEXP=keep or MEMEXP=reopen")
	}
	s, err := Open(filepath.Join(t.TempDir(), "ops.duckdb"), 256)
	if err != nil {
		t.Fatal(err)
	}
	defer s.Close()
	// One bulk insert stands in for the day's traffic: varied paths,
	// agents, networks and visitors, so the aggregates have real work.
	if _, err := s.db.Exec(`INSERT INTO hit
		(ts, door, method, url, repo, status, bytes, elapsed_ms, stream, addr, country, ua, referrer,
		 account, claimed, verb, client, version, asn, network, visitor)
		SELECT now() - INTERVAL (i % 86000) SECOND, CASE WHEN i % 5 = 0 THEN 'sync' ELSE 'web' END,
			'GET', '/r/repo' || (i % 300) || '/file/' || (i % 5000), 'repo' || (i % 300),
			200, i % 100000, i % 900, false, '203.0.113.' || (i % 250), 'US',
			'Mozilla/5.0 (X11; Linux x86_64) agent/' || (i % 400), '', '', '', '', '', '',
			CAST(i % 600 AS UINTEGER), 'net' || (i % 600), CAST(i % 90000 AS UBIGINT)
		FROM range(1300000) t(i)`); err != nil {
		t.Fatal(err)
	}
	fmt.Println("loaded 1.3M hits, resident", residentMB())
	now := time.Now()
	for i := 1; i <= 400; i++ {
		for j := 0; j < 256; j++ {
			s.Hit(obs.Hit{At: now, Door: obs.DoorWeb, URL: fmt.Sprintf("/r/repo%d/x", j), Repo: "repo", Status: 200,
				UA: fmt.Sprintf("Mozilla/5.0 agent/%d", j), Visitor: uint64(j), Addr: "203.0.113.7", ASN: uint32(j), Network: "n"})
		}
		s.Flush()
		if _, err := s.Traffic(obs.TrafficQuery{Now: time.Now(), Range: obs.RangeNamed("24h")}); err != nil {
			t.Fatal(err)
		}
		if _, err := s.History(obs.HistoryQuery{}); err != nil {
			t.Fatal(err)
		}
		if i%50 == 0 {
			fmt.Printf("iteration %d resident %s\n", i, residentMB())
		}
		if mode == "reopen" && i%100 == 0 {
			if err := s.Reopen(); err != nil {
				t.Fatal(err)
			}
			fmt.Printf("iteration %d after reopen resident %s\n", i, residentMB())
		}
	}
}