package ingest
import (
"fmt"
"os"
"os/exec"
"path/filepath"
"runtime"
"sort"
"strings"
"syscall"
"testing"
"time"
"craftdesign.group/quarry/internal/zitcli"
)
// maxRSS is the test process's peak resident set so far, in MiB.
func maxRSS(t *testing.T) float64 {
t.Helper()
var ru syscall.Rusage
if err := syscall.Getrusage(syscall.RUSAGE_SELF, &ru); err != nil {
t.Fatal(err)
}
if runtime.GOOS == "darwin" {
return float64(ru.Maxrss) / (1 << 20) // bytes
}
return float64(ru.Maxrss) / (1 << 10) // KiB
}
// countZit puts a zit on PATH that logs each invocation's verb to the
// file $ZIT_CALL_LOG names, when it names one, and makes each diff take
// $ZIT_SLOW seconds longer, when that is set, before running the real
// one.
func countZit(t *testing.T) {
t.Helper()
real, err := exec.LookPath("zit")
if err != nil {
t.Skip("zit not on PATH")
}
dir := t.TempDir()
shim := "#!/bin/sh\nif [ -n \"$ZIT_CALL_LOG\" ]; then echo \"$1\" >> \"$ZIT_CALL_LOG\"; fi\n" +
"if [ -n \"$ZIT_SLOW\" ] && [ \"$1\" = diff ]; then sleep \"$ZIT_SLOW\"; fi\nexec " + real + " \"$@\"\n"
if err := os.WriteFile(filepath.Join(dir, "zit"), []byte(shim), 0o755); err != nil {
t.Fatal(err)
}
t.Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH"))
}
// A live round's cost for a tree check-in does not grow with the tree:
// two later check-ins of one file each cost two zit calls apiece (their
// changed files and their line counts) and no artifact is read one by
// one. A few thousand files, so it runs on every check-in.
func TestALiveRoundOnATreeCostsTwoCallsPerCheckin(t *testing.T) {
treeRoundCost(t, 3_000)
}
// The same, measured on 50,000 files. About three minutes, nearly all
// of it building the history, and CI runs without -short: asked for by
// name (QUARRY_TREE_COST=1), not run on every check-in.
func TestALiveRoundOnALargeTreeCostsAFewCallsPerCheckin(t *testing.T) {
if testing.Short() || os.Getenv("QUARRY_TREE_COST") == "" {
t.Skip("builds a 50,000-file history; set QUARRY_TREE_COST=1 to run it")
}
treeRoundCost(t, 50_000)
}
func treeRoundCost(t *testing.T, files int) {
t.Helper()
countZit(t)
r := newTreeRig(t)
for i := range files {
p := filepath.Join(r.git, fmt.Sprintf("m%02d", i%40), fmt.Sprintf("s%03d", i%997), fmt.Sprintf("f%05d.txt", i))
if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(p, []byte(fmt.Sprintln("file", i)), 0o644); err != nil {
t.Fatal(err)
}
}
r.in(t, r.git, "git", "add", "-A")
r.in(t, r.git, "git", "commit", "-qm", "many files")
r.commits = 1
// The import's own sub-manifest choice, as a real import makes it.
r.in(t, r.git, "zit", "import-git", r.git, r.imported, "--quiet")
r.push(t, true)
start := time.Now()
if _, err := r.p.Sync(); err != nil {
t.Fatal(err)
}
t.Logf("first push (one root check-in, %d files): live round %s", files, time.Since(start).Round(time.Millisecond))
for i := range 2 {
p := filepath.Join(r.git, "m00", "s000", fmt.Sprintf("touched%d.txt", i))
if err := os.WriteFile(p, []byte("touched\n"), 0o644); err != nil {
t.Fatal(err)
}
r.in(t, r.git, "git", "add", "-A")
r.in(t, r.git, "git", "commit", "-qm", fmt.Sprint("touch ", i))
r.commits++
}
r.in(t, r.git, "zit", "import-git", r.git, r.imported, "--into", "--quiet")
src := zitcli.New(r.imported)
srcTip, _, err := src.Info(t.Context(), "trunk")
if err != nil {
t.Fatal(err)
}
raw, err := src.ArtifactRaw(t.Context(), srcTip, 1<<20)
if err != nil || !strings.Contains(string(raw), "\nS ") {
t.Fatalf("rig: the tip is not a tree check-in (%v):\n%.400s", err, raw)
}
r.push(t, false)
t.Setenv("ZIT_CALL_LOG", filepath.Join(t.TempDir(), "calls"))
rssBefore := maxRSS(t)
start = time.Now()
if _, err := r.p.Sync(); err != nil {
t.Fatal(err)
}
wall := time.Since(start)
rssAfter := maxRSS(t)
logFile := os.Getenv("ZIT_CALL_LOG")
t.Setenv("ZIT_CALL_LOG", "")
raw, _ = os.ReadFile(logFile)
verbs := strings.Fields(string(raw))
byVerb := map[string]int{}
for _, v := range verbs {
byVerb[v]++
}
keys := make([]string, 0, len(byVerb))
for k := range byVerb {
keys = append(keys, k)
}
sort.Strings(keys)
var parts []string
for _, k := range keys {
parts = append(parts, fmt.Sprintf("%s=%d", k, byVerb[k]))
}
t.Logf("live round for 2 one-file tree check-ins on a %d-file tree: %d zit calls (%s), %s wall, peak RSS %.0f MiB before, %.0f MiB after",
files, len(verbs), strings.Join(parts, " "), wall.Round(time.Millisecond), rssBefore, rssAfter)
tip, _, err := r.cli.Info(t.Context(), "trunk")
if err != nil {
t.Fatal(err)
}
if _, _, ok, err := r.ix.CheckinByPrefix(tip); err != nil || !ok {
t.Errorf("FAIL: the tip %.12s is not indexed (%v)", tip, err)
}
stats, err := r.ix.ChangeStats([]string{tip}, "")
if err != nil || stats[tip].Files != 1 {
t.Errorf("FAIL: the tip records %+v (%v), want one changed file", stats[tip], err)
}
// Per check-in: the changed files and their counts. The rest of the
// round (frames, the gap probe) is the round's, whatever it carries.
if per := byVerb["diff"]; per != 2*2 {
t.Errorf("FAIL: %d zit diff calls for 2 check-ins, want 2 each", per)
}
if len(verbs) > 2*2+2 {
t.Errorf("FAIL: %d zit calls in the round (%s), want the 4 diffs and the round's own few", len(verbs), strings.Join(parts, " "))
}
if n := byVerb["artifact"]; n > 0 {
t.Errorf("FAIL: the round read %d artifacts one by one: a tree walk", n)
}
if !t.Failed() {
t.Log("ok")
}
}