Artifact [9700517602]

Artifact 9700517602f0f539169b2f23f35a4d80b1f0f210d732817b4e479175ee0bb0ac:

• File repo_content_cache_test.go — part of check-in [af6431af27] at 2026-10-26 14:05:59 on branch main — test: consolidate root white-box tests by source file

  Merge the root package's white-box test files so each mirrors the source file it exercises (checkout, fossil, horizon, repo_checkout, repo_history, repo_merge, repo_sync, repo_ticket, repo_extras, required_content, transport), and rename single files to match their source (repo_content_cache_test.go, repo_admin_test.go, required_content_checkout_test.go). Test bodies are unchanged; only imports are consolidated. (user: [email protected], size: 11194) [annotate] [blame] [check-ins using] [more...]

• File repo_read_blob_test.go — part of check-in [d367012009] at 2026-10-25 23:10:47 on branch feat/read-blob-content-cache — feat: read an artifact by hash and cache content on Repo

  Repo.ReadBlob expands one artifact by its full hash after the availability check ReadFile applies, and names an unknown hash with ErrArtifactNotFound.

  Repo.SetContentCache attaches a byte-bounded content.Cache that ReadFile, ReadFileAt, ReadBlob, ListFiles, Diff and Annotate share; check-in parses are kept in its derived memo, so reads at one check-in parse the manifest once. Default off. A read that finds the DB's finished-write count moved swaps in an empty cache, so a purged, compacted or elided artifact, and a rid SQLite reuses after a purge, is never served from it. Writes by another process are not seen; the method's doc says so.

  Tracked in #492. (user: [email protected], size: 11194) [annotate] [blame] [check-ins using]

package libfossil

import (
	"context"
	"errors"
	"fmt"
	"path/filepath"
	"reflect"
	"strings"
	"sync"
	"testing"

	"github.com/danmestas/go-libfossil/internal/content"
	"github.com/danmestas/go-libfossil/internal/hash"
	"github.com/danmestas/go-libfossil/internal/shun"
)

// newReadBlobRepo commits three revisions of a few files and returns the
// repository with the check-in rids, oldest first.
func newReadBlobRepo(t *testing.T) (*Repo, []int64) {
	t.Helper()
	r, err := Create(filepath.Join(t.TempDir(), "r.fossil"), CreateOpts{User: "test"})
	if err != nil {
		t.Fatal(err)
	}
	t.Cleanup(func() { r.Close() })
	var rids []int64
	parent := int64(0)
	for rev := 0; rev < 3; rev++ {
		var files []FileToCommit
		for _, name := range []string{"a.txt", "b.txt", "dir/c.txt", "dir/d.txt"} {
			body := fmt.Sprintf("%s line one\nshared\n%s rev %d\n", name, name, rev)
			files = append(files, FileToCommit{Name: name, Content: []byte(body)})
		}
		rid, _, err := r.Commit(CommitOpts{User: "test", ParentID: parent, Files: files})
		if err != nil {
			t.Fatal(err)
		}
		rids = append(rids, rid)
		parent = rid
	}
	return r, rids
}

// cacheStats is the attached cache's statistics; the test fails if none is.
func cacheStats(t *testing.T, r *Repo) content.CacheStats {
	t.Helper()
	c := r.liveCache()
	if c == nil {
		t.Fatal("no content cache attached")
	}
	return c.Stats()
}

// ReadBlob of a hash ListFiles names returns what ReadFile of the path does,
// with and without a content cache.
func TestReadBlobMatchesReadFile(t *testing.T) {
	r, rids := newReadBlobRepo(t)
	for _, cached := range []bool{false, true} {
		if cached {
			r.SetContentCache(1 << 20)
		}
		for _, rid := range rids {
			files, err := r.ListFiles(rid)
			if err != nil {
				t.Fatal(err)
			}
			for _, f := range files {
				want, err := r.ReadFile(rid, f.Name)
				if err != nil {
					t.Fatal(err)
				}
				got, err := r.ReadBlob(f.UUID)
				if err != nil {
					t.Fatalf("cached=%v ReadBlob(%s): %v", cached, f.Name, err)
				}
				if string(got) != string(want) {
					t.Fatalf("cached=%v %s: ReadBlob %q, ReadFile %q", cached, f.Name, got, want)
				}
			}
		}
	}
}

// ReadBlob refuses an empty or malformed hash before touching the store, and
// names an unknown hash as an unknown artifact.
func TestReadBlobRejects(t *testing.T) {
	r, _ := newReadBlobRepo(t)
	for _, uuid := range []string{"", "abc", strings.Repeat("A", 40), strings.Repeat("g", 64)} {
		if _, err := r.ReadBlob(uuid); err == nil {
			t.Fatalf("ReadBlob(%q) succeeded", uuid)
		}
	}
	unknown := hash.SHA1([]byte("never committed\n"))
	if _, err := r.ReadBlob(unknown); !errors.Is(err, ErrArtifactNotFound) {
		t.Fatalf("ReadBlob(unknown) = %v, want ErrArtifactNotFound", err)
	}
}

// Below a horizon ReadBlob fails as ReadFile does: the blob is an elided
// phantom, and both say so the same way, with and without a cache.
func TestReadBlobBelowHorizon(t *testing.T) {
	f := newHorizonFixture(t)
	hc, err := NewHorizonCone(nil, nil, Horizon{Checkin: f.c2})
	if err != nil {
		t.Fatal(err)
	}
	target, _, err := Clone(context.Background(), filepath.Join(t.TempDir(), "t.fossil"),
		f.peer, CloneOpts{StoreCone: hc, PullPhantoms: true})
	if err != nil {
		t.Fatal(err)
	}
	defer target.Close()
	rid1, err := target.ResolveVersion(f.c1)
	if err != nil {
		t.Fatal(err)
	}
	old := hash.SHA1([]byte("old a\n"))
	for _, cached := range []bool{false, true} {
		if cached {
			target.SetContentCache(1 << 20)
		}
		_, fileErr := target.ReadFile(rid1, "a")
		_, blobErr := target.ReadBlob(old)
		for _, err := range []error{fileErr, blobErr} {
			if err == nil || !strings.Contains(err.Error(), "elided") ||
				!strings.Contains(err.Error(), "widen/deepen") {
				t.Fatalf("cached=%v below-horizon read: %v", cached, err)
			}
		}
		if got, err := target.ReadBlob(hash.SHA1([]byte("shared b\n"))); err != nil ||
			string(got) != "shared b\n" {
			t.Fatalf("cached=%v held blob: %q, %v", cached, got, err)
		}
	}
}

// With a cache, a second path at the same check-in is served from the parse
// the first read kept: no manifest is expanded or parsed again, and the only
// miss is the new file's own blob.
func TestContentCacheReusesManifest(t *testing.T) {
	r, rids := newReadBlobRepo(t)
	r.SetContentCache(1 << 20)
	tip := rids[len(rids)-1]
	if _, err := r.ReadFile(tip, "a.txt"); err != nil {
		t.Fatal(err)
	}
	before := cacheStats(t, r)
	if _, err := r.ReadFileAt("tip", "dir/c.txt"); err != nil {
		t.Fatal(err)
	}
	after := cacheStats(t, r)
	if after.DerivedHits <= before.DerivedHits {
		t.Fatalf("manifest parse not reused: derived hits %d -> %d",
			before.DerivedHits, after.DerivedHits)
	}
	if got := after.Misses - before.Misses; got != 1 {
		t.Fatalf("second read missed %d times, want 1 (its own blob)", got)
	}
	if _, err := r.ListFiles(tip); err != nil {
		t.Fatal(err)
	}
	if got := cacheStats(t, r).Misses - after.Misses; got != 0 {
		t.Fatalf("ListFiles of a read check-in missed %d times", got)
	}
}

// Diff and Annotate answer the same with a cache as without.
func TestContentCacheDiffAnnotateUnchanged(t *testing.T) {
	r, rids := newReadBlobRepo(t)
	read := func() ([]DiffEntry, []AnnotatedLine) {
		d, err := r.Diff(rids[0], rids[2], "")
		if err != nil {
			t.Fatal(err)
		}
		a, err := r.Annotate(AnnotateOpts{FilePath: "dir/c.txt", StartRID: rids[2]})
		if err != nil {
			t.Fatal(err)
		}
		return d, a
	}
	wantDiff, wantAnn := read()
	r.SetContentCache(1 << 20)
	for range 2 {
		gotDiff, gotAnn := read()
		if !reflect.DeepEqual(gotDiff, wantDiff) || !reflect.DeepEqual(gotAnn, wantAnn) {
			t.Fatal("cached Diff or Annotate differs from uncached")
		}
	}
	if cacheStats(t, r).Hits == 0 {
		t.Fatal("Diff and Annotate did not use the cache")
	}
}

// Compaction elides blobs the cache already holds. A read after it refuses as
// an uncached read would, rather than serving the bytes the cache kept.
func TestContentCacheNotStaleAfterCompaction(t *testing.T) {
	f := newCompactFixture(t)
	r := f.target
	r.SetContentCache(1 << 20)
	files, err := r.ListFiles(f.parent)
	if err != nil {
		t.Fatal(err)
	}
	uuid := ""
	for _, file := range files {
		if file.Name == "zzz/b" {
			uuid = file.UUID
		}
	}
	if got, err := r.ReadFile(f.parent, "zzz/b"); err != nil || string(got) != "outside\n" {
		t.Fatalf("before compaction: %q, %v", got, err)
	}
	if _, err := r.ReadBlob(uuid); err != nil {
		t.Fatal(err)
	}
	narrow, err := NewCone([]string{"src"}, nil)
	if err != nil {
		t.Fatal(err)
	}
	if err := r.SetStoreCone(narrow); err != nil {
		t.Fatal(err)
	}
	if n, err := r.CompactStore(context.Background()); err != nil || n == 0 {
		t.Fatalf("compaction = %d, %v", n, err)
	}
	if got, err := r.ReadFile(f.parent, "zzz/b"); err == nil {
		t.Fatalf("ReadFile served elided content %q", got)
	}
	if got, err := r.ReadBlob(uuid); err == nil {
		t.Fatalf("ReadBlob served elided content %q", got)
	}
	if got, err := r.ReadFile(f.parent, "src/a"); err != nil || string(got) != "inside\n" {
		t.Fatalf("in-cone read after compaction: %q, %v", got, err)
	}
}

// A shunned and purged artifact is gone, and the cache that held it does not
// bring it back: not the purged check-in's manifest, which a read through it
// would otherwise still resolve paths against, and not under its rid, which
// SQLite hands to the next blob stored once the newest row is deleted.
func TestContentCacheNotStaleAfterPurge(t *testing.T) {
	r, rids := newReadBlobRepo(t)
	r.SetContentCache(1 << 20)
	tip := rids[len(rids)-1]
	var newest int64
	if err := r.DB().QueryRow("SELECT max(rid) FROM blob").Scan(&newest); err != nil {
		t.Fatal(err)
	}
	if newest != tip {
		t.Fatalf("newest blob is rid %d, want the tip manifest %d", newest, tip)
	}
	tipUUID, err := r.UUIDFromRID(tip)
	if err != nil {
		t.Fatal(err)
	}
	if _, err := r.ReadFile(tip, "a.txt"); err != nil {
		t.Fatal(err)
	}
	if _, err := r.ReadBlob(tipUUID); err != nil {
		t.Fatal(err)
	}
	if err := shun.Add(r.DB(), tipUUID, "test"); err != nil {
		t.Fatal(err)
	}
	if _, err := shun.Purge(r.DB()); err != nil {
		t.Fatal(err)
	}
	if got, err := r.ReadBlob(tipUUID); !errors.Is(err, ErrArtifactNotFound) {
		t.Fatalf("ReadBlob after purge = %q, %v; want ErrArtifactNotFound", got, err)
	}
	if got, err := r.ReadFile(tip, "a.txt"); err == nil {
		t.Fatalf("ReadFile through a purged manifest served %q", got)
	}
	body := []byte("stored after the purge\n")
	next, nextUUID, err := r.Commit(CommitOpts{User: "test", ParentID: rids[1],
		Files: []FileToCommit{{Name: "fresh.txt", Content: body}}})
	if err != nil {
		t.Fatal(err)
	}
	files, err := r.ListFiles(next)
	if err != nil {
		t.Fatal(err)
	}
	for _, f := range files {
		got, err := r.ReadBlob(f.UUID)
		if err != nil || hash.ContentHash(got, f.UUID) != f.UUID {
			t.Fatalf("ReadBlob(%s) = %q, %v: not the named content", f.Name, got, err)
		}
	}
	got, err := r.ReadBlob(nextUUID)
	if err != nil || hash.ContentHash(got, nextUUID) != nextUUID {
		t.Fatalf("ReadBlob(new manifest) = %v: not the named content", err)
	}
}

// The cache holds no more expanded content than it was given, however many
// distinct blobs are read through it.
func TestContentCacheBounded(t *testing.T) {
	r, err := Create(filepath.Join(t.TempDir(), "r.fossil"), CreateOpts{User: "test"})
	if err != nil {
		t.Fatal(err)
	}
	defer r.Close()
	var files []FileToCommit
	for i := range 64 {
		body := strings.Repeat(fmt.Sprintf("file %d\n", i), 128)
		files = append(files, FileToCommit{Name: fmt.Sprintf("f%02d", i), Content: []byte(body)})
	}
	rid, _, err := r.Commit(CommitOpts{User: "test", Files: files})
	if err != nil {
		t.Fatal(err)
	}
	const budget = 8 << 10
	r.SetContentCache(budget)
	for _, f := range files {
		if _, err := r.ReadFile(rid, f.Name); err != nil {
			t.Fatal(err)
		}
		if s := cacheStats(t, r); s.Size > budget || s.MaxSize != budget {
			t.Fatalf("cache holds %d bytes of a %d budget (max %d)", s.Size, budget, s.MaxSize)
		}
	}
	r.SetContentCache(0)
	if r.liveCache() != nil {
		t.Fatal("SetContentCache(0) left a cache attached")
	}
	if _, err := r.ReadFile(rid, "f00"); err != nil {
		t.Fatal(err)
	}
}

// Reads may run while the cache is attached, replaced and detached and while
// the repository is written; run under -race.
func TestContentCacheConcurrentReads(t *testing.T) {
	r, rids := newReadBlobRepo(t)
	tip := rids[len(rids)-1]
	files, err := r.ListFiles(tip)
	if err != nil {
		t.Fatal(err)
	}
	var wg sync.WaitGroup
	errs := make(chan error, 64)
	for g := range 4 {
		wg.Add(1)
		go func() {
			defer wg.Done()
			for i := range 20 {
				f := files[(g+i)%len(files)]
				a, err := r.ReadFile(tip, f.Name)
				if err != nil {
					errs <- err
					return
				}
				b, err := r.ReadBlob(f.UUID)
				if err != nil || string(a) != string(b) {
					errs <- fmt.Errorf("%s: ReadBlob %q, %v; ReadFile %q", f.Name, b, err, a)
					return
				}
			}
		}()
	}
	wg.Add(1)
	go func() {
		defer wg.Done()
		for i := range 20 {
			r.SetContentCache(int64(i%3) << 16)
		}
		if _, _, err := r.Commit(CommitOpts{User: "test", ParentID: tip,
			Files: []FileToCommit{{Name: "new.txt", Content: []byte("new\n")}}}); err != nil {
			errs <- err
		}
	}()
	wg.Wait()
	close(errs)
	for err := range errs {
		t.Fatal(err)
	}
}