Artifact 29a43d3ee08ab013213e612cfe81fcf0a0bcca860cc6d44e7dfbd04827031290:
- File
internal/verify/rebuild_manifest.go
— part of check-in
[0fd2a507b8]
at
2026-07-24 15:27:03
on branch trunk
— perf: route history-walking expansions through content.Cache (#151) (#153)
The buffer-reuse fix in #145/#150 cut the cost of expanding one delta chain once. It did nothing for callers that expand many versions of the same artifact in a single operation: each bare content.Expand still replayed the whole O(depth) chain, so a walk over N overlapping revisions cost O(N^2) blob reads even though content.Cache already existed for exactly this pattern.
Audited every non-test content.Expand caller. Two access patterns genuinely repeat over overlapping delta chains and are now routed through a per-operation content.Cache:
- annotate walks every revision of one file along the parent chain. content_deltify stores each older file blob as a delta against its successor, so the per-revision expansions are all walks of the same few chains. One cache threaded through the walk (file blob + manifest listing via new manifest.ListFilesCached/GetManifestCached, both nil-passthrough so existing callers are unchanged) turns it linear. Measured on a deltified 150-revision file: 1.60s -> 0.65s (2.5x), 764k -> 308k allocs; the win grows with history depth.
- verify.Verify and verify.Rebuild sweep every blob more than once (Rebuild: checkBlobs + rebuildManifests + rebuildTags x2). A single cache shared across the pass walks each chain once instead of once per sweep and amortizes overlap within each sweep. Measured on a 150-revision two-file repo: Rebuild 3.34s -> 0.068s (48x), 4.59M -> 98k allocs.
Callers that only ever touch the tip or make a single pass over distinct blobs (branch create, merge-config load, R-card / delta manifest build, MergeParentFiles, crosslinkSingle, shun undelta, pairwise Diff) are left on bare Expand, each with a comment stating why a cache cannot amortize their access.
No behavior change: expansion output is byte-identical, verified across the modernc and ncruces drivers with REQUIRE_FOSSIL_BIN=1 cross-checks and the DST suite. (user: [email protected], size: 6124) [annotate] [blame] [check-ins using]
package verify import ( "fmt" "github.com/danmestas/go-libfossil/db" "github.com/danmestas/go-libfossil/internal/blob" "github.com/danmestas/go-libfossil/internal/content" "github.com/danmestas/go-libfossil/internal/deck" libfossil "github.com/danmestas/go-libfossil/internal/fsltype" "github.com/danmestas/go-libfossil/internal/repo" ) // rebuildManifests walks all non-phantom blobs, parses checkin manifests, // and inserts event/plink/mlink/filename rows. func rebuildManifests(r *repo.Repo, tx *db.Tx, report *Report, cache *content.Cache) error { if r == nil { panic("rebuildManifests: nil *repo.Repo") } if tx == nil { panic("rebuildManifests: nil *db.Tx") } if report == nil { panic("rebuildManifests: nil *Report") } entries, err := collectBlobEntries(tx) if err != nil { return err } for _, e := range entries { data, err := cache.Expand(tx, e.rid) if err != nil { report.BlobsSkipped++ continue // not expandable — corrupt, raw data blob, or phantom } d, err := deck.Parse(data) if err != nil { continue // not a manifest — normal for file blobs } if d.Type != deck.Checkin { continue } if err := rebuildCheckin(tx, e.rid, d, report, cache); err != nil { return fmt.Errorf("rebuildManifests rid=%d: %w", e.rid, err) } } return nil } // blobEntry holds a blob's rid and uuid for rebuild iteration. type blobEntry struct { rid libfossil.FslID uuid string } // collectBlobEntries reads all non-phantom blob rid/uuid pairs. func collectBlobEntries(q db.Querier) ([]blobEntry, error) { rows, err := q.Query("SELECT rid, uuid FROM blob WHERE size >= 0") if err != nil { return nil, fmt.Errorf("collectBlobEntries: %w", err) } defer rows.Close() var entries []blobEntry for rows.Next() { var e blobEntry if err := rows.Scan(&e.rid, &e.uuid); err != nil { return nil, fmt.Errorf("collectBlobEntries scan: %w", err) } entries = append(entries, e) } return entries, rows.Err() } // rebuildCheckin inserts event, plink, and mlink rows for one checkin manifest. func rebuildCheckin(tx *db.Tx, rid libfossil.FslID, d *deck.Deck, report *Report, cache *content.Cache) error { if tx == nil { panic("rebuildCheckin: nil *db.Tx") } if d == nil { panic("rebuildCheckin: nil *deck.Deck") } mtime := libfossil.TimeToJulian(d.D) // Insert event row if _, err := tx.Exec( "INSERT OR IGNORE INTO event(type, mtime, objid, user, comment) VALUES('ci', ?, ?, ?, ?)", mtime, rid, d.U, d.C, ); err != nil { return fmt.Errorf("event: %w", err) } // Insert plink rows for parent(s) if err := rebuildPlinks(tx, rid, d, mtime, report); err != nil { return err } // Insert mlink/filename rows for manifest F-cards. // Uses d.F directly (not expanded) — matches Fossil's mlink semantics. if err := rebuildMlinks(tx, rid, d, cache); err != nil { return err } return nil } // rebuildPlinks inserts plink rows for each parent in the manifest. func rebuildPlinks(tx *db.Tx, rid libfossil.FslID, d *deck.Deck, mtime float64, report *Report) error { for i, parentUUID := range d.P { parentRID, ok := blob.Exists(tx, parentUUID) if !ok { report.MissingRefs++ report.addIssue(Issue{ Kind: IssueMissingReference, RID: rid, UUID: parentUUID, Table: "plink", Message: fmt.Sprintf("rid=%d parent %s not found", rid, parentUUID), }) continue } isPrim := 0 if i == 0 { isPrim = 1 } if _, err := tx.Exec( "INSERT OR IGNORE INTO plink(pid, cid, isprim, mtime) VALUES(?, ?, ?, ?)", parentRID, rid, isPrim, mtime, ); err != nil { return fmt.Errorf("plink: %w", err) } } return nil } // rebuildMlinks inserts mlink and filename rows for files that are new or // changed relative to the primary parent. Fossil's rebuild only creates mlink // rows for files that differ from the parent checkin — unchanged files are // skipped. This matches fossil rebuild behavior. func rebuildMlinks(tx *db.Tx, rid libfossil.FslID, d *deck.Deck, cache *content.Cache) error { // Build parent file map (name → uuid) for comparison. parentFiles := buildParentFileMap(tx, d, cache) for _, f := range d.F { if f.UUID == "" { continue // deleted file in delta manifest } // Skip unchanged files — only create mlink for new or modified. if parentUUID, exists := parentFiles[f.Name]; exists && parentUUID == f.UUID { continue } fnid, err := rebuildEnsureFilename(tx, f.Name) if err != nil { return fmt.Errorf("filename %q: %w", f.Name, err) } fileRID, ok := blob.Exists(tx, f.UUID) if !ok { continue // file blob missing — phantom or not yet received } if _, err := tx.Exec( "INSERT OR IGNORE INTO mlink(mid, fid, fnid) VALUES(?, ?, ?)", rid, fileRID, fnid, ); err != nil { return fmt.Errorf("mlink: %w", err) } } return nil } // buildParentFileMap returns a map of filename→UUID from the primary parent's // manifest. Returns an empty map if there is no parent (initial checkin). func buildParentFileMap(tx *db.Tx, d *deck.Deck, cache *content.Cache) map[string]string { if len(d.P) == 0 { return nil // initial checkin — no parent } parentRID, ok := content.AvailableByUUID(tx, d.P[0]) // primary parent if !ok { return nil } parentData, err := cache.Expand(tx, parentRID) if err != nil { return nil } parentDeck, err := deck.Parse(parentData) if err != nil { return nil } m := make(map[string]string, len(parentDeck.F)) for _, f := range parentDeck.F { m[f.Name] = f.UUID } return m } // rebuildEnsureFilename ensures a filename row exists and returns its fnid. func rebuildEnsureFilename(tx *db.Tx, name string) (int64, error) { if tx == nil { panic("rebuildEnsureFilename: nil *db.Tx") } if name == "" { panic("rebuildEnsureFilename: empty name") } var fnid int64 err := tx.QueryRow("SELECT fnid FROM filename WHERE name=?", name).Scan(&fnid) if err == nil { return fnid, nil } result, err := tx.Exec("INSERT INTO filename(name) VALUES(?)", name) if err != nil { return 0, fmt.Errorf("insert filename %q: %w", name, err) } return result.LastInsertId() }