Artifact 3757803a5e7bf22ea18e4172c7d3161c81b97050754767a3b44b8c11b4cc9f54:
- File
internal/content/cache_chain_test.go
— part of check-in
[8b16c91f5c]
at
2026-07-28 04:03:58
on branch trunk
— perf: stop clone re-seeking blob.uuid per F-card and over-allocating delta output (#187) (#212)
Cloning the Fossil SCM corpus (67.6k blobs) took 293-302 s at 1251-1287 MB against canonical fossil's 20.8 s at 205 MB. A CPU and allocation profile of one clone found the cost was not delta expansion or hash verification.
27.2% of all CPU was a single un-memoized statement. content.AvailabilityCache memoizes the delta-chain walk but not the "SELECT rid FROM blob WHERE uuid=?" that starts it, so manifest.missingCheckinRefs re-seeked the blob.uuid index once per F-card of every check-in -- the same file hash resolved again for every revision that did not change it. That index is far larger than SQLite's page cache, so each seek was three page reads: pread syscalls were 34.7% of total samples. The walk it guards was under 1% of its own cost.
Separately, materializing 8.27 GB of content allocated 360 GB. delta.ApplyInto starts at a 64 KiB cap deliberately -- the target length is a wire claim and a few bytes can declare 4 GiB -- and then climbs to the real size through append's ~1.25x large-slice growth, ~3.4x the target in discarded intermediates, leaving slack that content.Cache.clip copied away again (203.9 GB and 59.8 GB respectively).
Three changes:
- Memoize uuid -> rid in AvailabilityCache. Hits only. Within one crosslink run a hit cannot go stale, because a sweep and a cascade only ever add blob rows; a miss can, because ridOrPhantom and blob.StorePhantom create rows mid-run, and remembering an absence would leave an artifact deferred after the content it waits on arrived. The scope that makes this sound is now stated on the type, and TestAvailabilityCacheMustNotSpanAPurge pins its one boundary: shun.Purge deletes blob rows, so a cache must not span one.
- Size delta.ApplyInto's output exactly. scanOutputSize walks the command stream first without allocating, checking every copy against bytes the source holds and every insert against bytes the delta carries, so the size it returns is a measurement rather than the header's claim. Any stream it cannot walk to a clean terminator falls back to the existing growth path, so malformed and hostile deltas allocate and fail exactly as before. The buffer now arrives tightly sized, which also makes clip's copy a no-op.
- Cut crosslinkCacheBytes from 256 to 128 MiB. Each byte of that budget costs roughly five bytes of peak RSS -- GOGC doubles it into the heap goal, and the scavenger is paced off GC cycles, of which a faster sweep runs fewer. At 256 MiB the first two changes gave 205.7 s but 1473 MB, faster at the cost of memory. The measured curve (32/64/128/256 MiB: 369.6 s/318 MB, 292.3 s/447 MB, 249.4 s/812 MB, 205.7 s/1473 MB) puts 128 MiB where both axes improve.
Result on the same corpus: 245.4 s at 780 MB, from 293-302 s at 1251-1287 MB -- 17% less wall time and 39% less peak RSS, with instructions retired down from 3.37T to 2.48T. The gap to canonical narrows from 14.1x/6.3x to 11.8x/3.8x.
Derived tables are unchanged against an unmodified build cloning the same server: blob 67530, event 27099 (identical per type), mlink 104919, plink 23741, tagxref 51766, leaf 1426, filename 1496. The corpus parity gate passes.
What remains is structural rather than a missing optimization: a clone expands every blob twice, once in the receive-time phantom-fill cascade and once in the final whole-repository sweep, where canonical crosslinks each artifact from content it already holds in hand at receive time. Cache.Expand splits 45.4 s / 35.3 s across those two passes. Collapsing them is a change to the receive path and belongs in its own issue. (user: [email protected], size: 4764) [annotate] [blame] [check-ins using]
package content import ( "bytes" "database/sql" "fmt" "strings" "testing" "github.com/danmestas/go-libfossil/db" "github.com/danmestas/go-libfossil/internal/blob" libfossil "github.com/danmestas/go-libfossil/internal/fsltype" ) // countingQuerier forwards to an inner Querier and counts the blob-content // reads, the single statement that dominates delta-chain expansion. type countingQuerier struct { inner db.Querier blobLoads int deltaWalks int availSteps int ridLookups int } func (c *countingQuerier) Exec(q string, args ...any) (sql.Result, error) { return c.inner.Exec(q, args...) } func (c *countingQuerier) QueryRow(q string, args ...any) *sql.Row { switch { case strings.HasPrefix(q, "SELECT content, size FROM blob"): c.blobLoads++ case strings.HasPrefix(q, "SELECT srcid FROM delta"): c.deltaWalks++ case strings.HasPrefix(q, "SELECT b.size, d.rid IS NOT NULL, d.srcid"): c.availSteps++ case strings.HasPrefix(q, "SELECT rid FROM blob WHERE uuid"): c.ridLookups++ } return c.inner.QueryRow(q, args...) } func (c *countingQuerier) Query(q string, args ...any) (*sql.Rows, error) { return c.inner.Query(q, args...) } // buildDeltaChain stores chainLen versions of a similar body and deltifies // each one against its successor, so version 0 sits at the deep end of a // chain rooted at the last version — the shape a real Fossil repository has // after content_deltify has rewritten older blobs as deltas of newer ones. func buildDeltaChain(t *testing.T, d *db.DB, chainLen int) ([]libfossil.FslID, [][]byte) { t.Helper() rids := make([]libfossil.FslID, chainLen) bodies := make([][]byte, chainLen) for i := range rids { var body bytes.Buffer for line := 0; line < 300; line++ { if line == i { fmt.Fprintf(&body, "line %04d: version %d\n", line, i) } else { fmt.Fprintf(&body, "line %04d: the quick brown fox jumps over the lazy dog\n", line) } } bodies[i] = body.Bytes() rid, _, err := blob.Store(d, bodies[i]) if err != nil { t.Fatalf("store version %d: %v", i, err) } rids[i] = rid } if err := d.WithTx(func(tx *db.Tx) error { for i := 0; i < chainLen-1; i++ { if _, err := Deltify(tx, rids[i], rids[i+1]); err != nil { return fmt.Errorf("deltify %d against %d: %w", i, i+1, err) } } return nil }); err != nil { t.Fatalf("build chain: %v", err) } var deltas int if err := d.QueryRow("SELECT count(*) FROM delta").Scan(&deltas); err != nil { t.Fatalf("count delta: %v", err) } if deltas != chainLen-1 { t.Fatalf("delta rows = %d, want %d — chain was not built", deltas, chainLen-1) } return rids, bodies } // TestCache_MemoizesChainInteriors pins the property Crosslink depends on: // walking a delta chain once must leave every node it materialized in the // cache, so expanding the whole chain costs a number of blob reads linear in // the chain length rather than quadratic. func TestCache_MemoizesChainInteriors(t *testing.T) { const chainLen = 40 d := setupTestDB(t) rids, bodies := buildDeltaChain(t, d, chainLen) q := &countingQuerier{inner: d} c := NewCache(8 << 20) // Ascending rid order is the order Crosslink sweeps in; rid[0] is the // deepest node, so its expansion materializes the entire chain. for i, rid := range rids { got, err := c.Expand(q, rid) if err != nil { t.Fatalf("Expand rid %d (version %d): %v", rid, i, err) } if !bytes.Equal(got, bodies[i]) { t.Fatalf("version %d: content mismatch", i) } } // Without interior memoization this is chainLen*(chainLen+1)/2 = 820. if q.blobLoads > 2*chainLen { t.Fatalf("blob content reads = %d, want <= %d for a %d-node chain", q.blobLoads, 2*chainLen, chainLen) } } // TestCache_ChainInteriorContentIsCorrect checks the memoized interiors are // served as their own content, not as a neighbour's. func TestCache_ChainInteriorContentIsCorrect(t *testing.T) { const chainLen = 12 d := setupTestDB(t) rids, bodies := buildDeltaChain(t, d, chainLen) c := NewCache(8 << 20) if _, err := c.Expand(d, rids[0]); err != nil { t.Fatalf("prime chain: %v", err) } for i := chainLen - 1; i >= 0; i-- { got, err := c.Expand(d, rids[i]) if err != nil { t.Fatalf("Expand version %d: %v", i, err) } if !bytes.Equal(got, bodies[i]) { t.Fatalf("version %d served wrong content", i) } } // Cached interiors must be independent buffers: mutating a returned // slice must not corrupt the next read of the same rid. first, err := c.Expand(d, rids[chainLen/2]) if err != nil { t.Fatal(err) } first[0] ^= 0xFF second, err := c.Expand(d, rids[chainLen/2]) if err != nil { t.Fatal(err) } if !bytes.Equal(second, bodies[chainLen/2]) { t.Fatal("mutating a returned slice corrupted the cached interior") } }