Artifact [533c50797e]

Artifact 533c50797e18476f2e266c139940a9c0daeec21edf53a914457536b11c03eb8b:

• File internal/highlight/gate_test.go — part of check-in [05dbba300d] at 2026-08-31 12:34:25 on branch trunk — collapse concurrent highlight misses into one tokenisation

  A read-through memo only helps once a value exists: ten cold readers of sqlite's where.c each took 0.95s against 0.42s for one, all tokenising the same immutable bytes. Safe because the value is a pure function of its key. x/sync was already in the module graph. (user: quarry, size: 3949) [annotate] [blame] [check-ins using]

package highlight

import (
	"strings"
	"sync"
	"sync/atomic"
	"testing"
)

// Highlighting cost tracks token density and was paid on every request:
// sqlite's src/btree.c, 407KB of dense C, took 490ms and rendered 2.3MB
// of HTML. Past MaxBytes the lexer is skipped so the page arrives.
func TestOversizeContentIsNotHighlighted(t *testing.T) {
	big := []byte(strings.Repeat("int x = 1; /* filler */\n", (MaxBytes/23)+64))
	if len(big) <= MaxBytes {
		t.Fatalf("fixture is %d bytes, not over the %d limit", len(big), MaxBytes)
	}
	if !TooBig(big) {
		t.Error("TooBig said no for content past the limit")
	}
	if html, ok := Code(big, "", "big.c"); ok || html != "" {
		t.Errorf("oversize content was highlighted (%d bytes of html)", len(html))
	}
	// Just under the limit still gets colour: the gate must not swallow
	// ordinary files.
	small := []byte("int x = 1;\n")
	if _, ok := Code(small, "", "small.c"); !ok {
		t.Error("a small C file was not highlighted")
	}
}

// A blob is immutable, so its highlighting is too. The second call must
// not tokenise again — and must return the same bytes.
func TestHighlightingIsCachedByContent(t *testing.T) {
	src := []byte("package main\n\nfunc main() { println(\"x\") }\n")
	first, ok := Code(src, "", "a.go")
	if !ok || first == "" {
		t.Fatal("first highlight failed")
	}
	cache.Lock()
	n := len(cache.m)
	cache.Unlock()
	second, ok2 := Code(src, "", "a.go")
	if !ok2 || second != first {
		t.Error("cached result differs from the first render")
	}
	cache.Lock()
	after := len(cache.m)
	cache.Unlock()
	if after != n {
		t.Errorf("a repeat of identical content added an entry (%d -> %d)", n, after)
	}
	// Same bytes, different filename, is a different lexer question.
	if _, _ = Code(src, "", "a.txt"); true {
		cache.Lock()
		defer cache.Unlock()
		if len(cache.m) == after {
			t.Error("the filename is not part of the cache key")
		}
	}
}

// The budget is in BYTES because entries are wildly uneven. Overflowing
// it must not grow without bound.
func TestCacheStaysWithinItsBudget(t *testing.T) {
	cache.Lock()
	cache.m, cache.bytes = map[string]string{}, 0
	cache.Unlock()
	cachePut("k", strings.Repeat("x", cacheBudget-1))
	cachePut("k2", strings.Repeat("y", 1024))
	cache.Lock()
	defer cache.Unlock()
	if cache.bytes > cacheBudget {
		t.Errorf("cache holds %d bytes, over the %d budget", cache.bytes, cacheBudget)
	}
}

// Concurrent readers of one cold key must tokenise ONCE. A read-through
// memo helps only after a value exists, so without collapsing the
// flight, N readers arriving together all miss and all do the same
// work — the machine does N tokenisations exactly when it can least
// afford to. Measured before singleflight: one cold request for
// sqlite's where.c took 0.42s and ten concurrent ones took 0.95s each.
func TestConcurrentMissesTokeniseOnce(t *testing.T) {
	cache.Lock()
	cache.m, cache.bytes = map[string]string{}, 0
	cache.Unlock()

	// Unique content, so this key is certainly cold.
	src := []byte("package p\n\n// " + t.Name() + "\nfunc f() { println(1) }\n")

	var calls atomic.Int64
	counted := func() (string, bool) {
		calls.Add(1)
		return code_(src, "", "x.go")
	}
	// Drive the same key through the same group the cache uses.
	key := cacheKey(src, "", "x.go")
	var wg sync.WaitGroup
	results := make([]string, 24)
	for i := range results {
		wg.Add(1)
		go func(i int) {
			defer wg.Done()
			v, _, _ := group.Do(key, func() (any, error) {
				html, _ := counted()
				cachePut(key, html)
				return html, nil
			})
			results[i], _ = v.(string)
		}(i)
	}
	wg.Wait()

	if n := calls.Load(); n == 0 {
		t.Fatal("nothing was highlighted")
	} else if n > 4 {
		t.Errorf("%d tokenisations for one key across 24 readers; the flight is not collapsing", n)
	}
	for i, r := range results {
		if r != results[0] {
			t.Fatalf("reader %d got a different result than reader 0", i)
		}
		if r == "" {
			t.Fatalf("reader %d got empty output", i)
		}
	}
}