Artifact [eda0500c46]

Artifact eda0500c46376f04de8b8f97d38c68de940da79f4e0abfb540503f027f2a7cdc:

• File internal/content/cone_reference_facts.go — part of check-in [b28c06506b] at 2026-10-22 02:57:50 on branch feat/horizon-cones — fix(content): classify sparse stores from crosslink tables, not the whole store

  Sparse classification read, inflated and delta-expanded every held blob, file content included, under a whole-store byte budget. On a clone of the Fossil SCM repository (67,894 blobs, 8.4 GB expanded) LoadAbsences failed at once with "byte budget exceeded", so no path-cone or content-horizon store of a real repository could classify its phantoms, and pull, verify and compaction failed with it.

  Reference facts now come from the tables wherever they provably stand in for raw cards. A blob the crosslinker recorded as no artifact and mlink names as a file version is never read. A completely linked check-in (event row, no orphan row, a primary parent edge) whose primary parent and baseline trees are themselves reproduced contributes its mlink rows as F cards and its plink, cherrypick and tagxref rows as references; by induction over the primary chain those rows give the same (name, file) pairs as its raw cards. Everything else is still read raw: unlinked and orphaned check-ins, roots, check-ins whose parent never arrived, other artifacts, and every check-in once the store has seen a sub-manifest, since trees are not crosslinked. Budgets are charged per round for the selected blobs and their delta ancestors only, and a base's bytes are released once its last delta is applied.

  A content horizon now reads only its held set's manifests and the baselines and nodes they reach, on demand, with one visited set and a byte budget scaled to the held set.

  On the Fossil SCM clone, cone {src} classifies in about 2 s at a 65 MiB peak heap, reading 8,655 of 67,894 blobs; the inside/outside sets equal the all-raw result, and Outside/Required agree on every artifact. (user: [email protected], size: 4488) [annotate] [blame] [check-ins using] [more...]

package content

import (
	"context"

	"github.com/danmestas/go-libfossil/db"
	"github.com/danmestas/go-libfossil/internal/cone"
	"github.com/danmestas/go-libfossil/internal/deck"
	libfossil "github.com/danmestas/go-libfossil/internal/fsltype"
)

// ConeReferenceFacts holds immutable reference facts for one database
// snapshot, without a cone key. The owner must validate that snapshot before
// using these facts against a database that may have changed.
//
// Loading names every artifact in the store, but reads raw bytes only where
// the crosslink tables cannot answer: never a file version the crosslinker
// found to be no artifact, and a check-in only when its link is incomplete
// or its tree is not reproduced by mlink (see answerable). It serves whole-store
// work (a pull's inventory, compaction, verification, crosslink), never a
// clone round. Retained decks are classification projections, not complete
// artifacts suitable for marshaling.
type ConeReferenceFacts struct {
	names      map[libfossil.FslID]string
	held       map[string]libfossil.FslID // Read and hash-verified.
	nodes      map[libfossil.FslID]*deck.Deck
	byUUID     map[string]*deck.Deck
	records    []coneReferenceRecord
	roots      []*deck.Deck
	unreadable []string
	// spine and files are linked check-ins' facts, from the tables: the
	// check-ins and their references, and their trees' (name, file) pairs.
	spine []string
	files []deck.FileCard
}

type coneReferenceRecord struct {
	uuid string
	deck *deck.Deck
}

func LoadConeReferenceFacts(ctx context.Context, q db.Querier) (*ConeReferenceFacts, error) {
	// Discovery is guarded before allocation. Counting only check-ins/trees
	// does not bound wiki, forum, cluster or file storage. Oversized
	// snapshots fail rather than allocate first and discover their budget
	// after parsing.
	facts, _, err := loadConeReferenceFactsBounded(ctx, q, defaultReferenceLimits.refs, true)
	return facts, err
}

// referenceLimits bounds one classification: its raw reference reads, and
// under a content horizon the tree walk of the held set, which is charged
// per check-in in that set rather than per artifact in the store.
type referenceLimits struct {
	refs           ReferenceBudget
	treePerCheckin int64
	treeMax        int64
}

// ExpandedBytes also pays for the delta chains a raw read must expand: a
// check-in whose parent never arrived is read raw, and in the Fossil SCM
// repository one such manifest sits 1,594 deltas from its full base, about
// 120 MiB of expansion. Bases are released as the chain advances, so this
// bounds work rather than memory. The tree budget scales with the held set
// so that a horizon's cost follows the history it keeps.
var defaultReferenceLimits = referenceLimits{
	refs: ReferenceBudget{
		Artifacts: coneReferenceNodes, StoredBytes: 64 << 20,
		InflatedBytes: 128 << 20, ExpandedBytes: 512 << 20, Cards: 1 << 20,
	},
	treePerCheckin: 1 << 20, treeMax: 4 << 30,
}

// Classify creates request-local membership; neither the facts nor previous
// classifications are changed. Maps of raw identities and decks are read-only.
func (facts *ConeReferenceFacts) Classify(
	ctx context.Context, c *cone.Cone,
) (*ConeReferences, error) {
	if facts == nil || ctx == nil {
		panic("content.ConeReferenceFacts.Classify: nil facts or context")
	}
	p := &ConeReferences{
		names: facts.names, held: facts.held, nodes: facts.nodes,
		outside: map[string]bool{}, inside: map[string]bool{},
		required: map[string]bool{}, tree: map[string]bool{},
	}
	if err := ctx.Err(); err != nil {
		return nil, err
	}
	if c == nil {
		return p, nil
	}
	for _, uuid := range facts.unreadable {
		p.required[uuid] = true
	}
	for _, uuid := range facts.spine {
		p.required[uuid] = true
	}
	for _, f := range facts.files {
		if c.Covers(f.Name) {
			p.inside[f.UUID] = true
		} else {
			p.outside[f.UUID] = true
		}
	}
	for _, record := range facts.records {
		if err := ctx.Err(); err != nil {
			return nil, err
		}
		d := record.deck
		if d.Type != deck.Submanifest {
			p.required[record.uuid] = true
		}
		for _, f := range d.F {
			if f.UUID == "" {
				continue
			}
			if c.Covers(f.Name) {
				p.inside[f.UUID] = true
			} else {
				p.outside[f.UUID] = true
			}
		}
		p.structural(d)
	}
	w := coneReferenceWalk{ctx: ctx, cone: c, refs: p, nodes: facts.byUUID, path: map[string]bool{}}
	for _, root := range facts.roots {
		if err := w.walk(root.S, "", 1, true); err != nil {
			return nil, err
		}
	}
	return p, nil
}