Artifact [e4fabed914]

Artifact e4fabed91463d59e059241081f26637620de29004411d8d2f2d69831c220c839:

• File internal/content/sparse.go — part of check-in [f1f4240f40] at 2026-10-22 11:18:55 on branch feat/horizon-cones — feat(sync): classify horizon stores locally against dense peers

  No peer speaks the horizon pragmas yet, so a content-horizon store receives whatever a dense peer sends (draft-fossil-horizon-00 §6.3). The received-content filter now applies the horizon's classification, read from internal/content rather than re-derived: blobs below the horizon become elided phantoms and are counted in the result's Elided. Blobs no held manifest names yet stay as unreferenced holdings, and retained delta bases stay transit (§4.2).

  A horizon store also acts on no cluster (§4.5). Crosslinking a cluster applies its tag but makes no member phantoms. The filter drops the cluster's content, and classification elides any phantom that carries the cluster tag. The client stops sending req-clusters. Path-cone-only stores keep today's cluster behaviour.

  Conformance cases run against go-libfossil's handler and fossil(1): clone and pull, deepen, demand fetch through UpdateContext, transit deltas, and cluster rules. (user: [email protected], size: 12028) [annotate] [blame] [check-ins using] [more...]

package content

import (
	"context"
	"database/sql"
	"errors"
	"fmt"

	"github.com/danmestas/go-libfossil/db"
	"github.com/danmestas/go-libfossil/internal/cone"
	libfossil "github.com/danmestas/go-libfossil/internal/fsltype"
)

// The store cone of draft-fossil-sparse-cone-00 §5 lives in a table of its
// own, created when a cone is set and dropped when it is cleared, so a dense
// repository's schema is untouched. `fossil rebuild` drops every table it
// does not know, which puts the cone in the disposable tier §5.2 recommends:
// losing it reclassifies every elided phantom as a transit phantom, and the
// store heals toward dense through ordinary synchronization.
const storeConeSchema = `CREATE TABLE IF NOT EXISTS storecone(
  id INTEGER PRIMARY KEY CHECK(id=1),
  cone BLOB NOT NULL
)`

// StoreCone returns the store cone, or nil when the store is dense. A cone
// this implementation cannot read -- a later serialization version, or bytes
// that do not parse -- is policy loss (§3.1, §5.2) and reads as dense, never
// as an error: the cost is bandwidth, not data.
func StoreCone(q db.Querier) (*cone.Cone, error) {
	if q == nil {
		panic("content.StoreCone: q must not be nil")
	}
	var n int
	if err := q.QueryRow(
		"SELECT count(*) FROM sqlite_master WHERE type='table' AND name='storecone'",
	).Scan(&n); err != nil {
		return nil, fmt.Errorf("content.StoreCone: %w", err)
	}
	if n == 0 {
		return nil, nil
	}
	var data []byte
	err := q.QueryRow("SELECT cone FROM storecone WHERE id=1").Scan(&data)
	if errors.Is(err, sql.ErrNoRows) {
		return nil, nil
	}
	if err != nil {
		return nil, fmt.Errorf("content.StoreCone: %w", err)
	}
	c, err := cone.Parse(data)
	if err != nil {
		return nil, nil
	}
	return c, nil
}

// SetStoreCone replaces the store cone; nil makes the store dense. Neither
// direction moves content: narrowing deletes nothing (§5.4), and widening
// only makes the newly covered elided phantoms wanted again, for the next
// synchronization to fetch. A structural horizon (draft-fossil-horizon-00
// §5) is refused: this store implements only the content horizon, and
// would otherwise hold a cone whose depth its classification ignores.
func SetStoreCone(q db.Querier, c *cone.Cone) error {
	if q == nil {
		panic("content.SetStoreCone: q must not be nil")
	}
	if c == nil {
		if _, err := q.Exec("DROP TABLE IF EXISTS storecone"); err != nil {
			return fmt.Errorf("content.SetStoreCone: %w", err)
		}
		return nil
	}
	if h, ok := c.Horizon(); ok && h.Depth() != cone.DepthContent {
		return fmt.Errorf("content.SetStoreCone: a %v horizon is not implemented", h.Depth())
	}
	if _, err := q.Exec(storeConeSchema); err != nil {
		return fmt.Errorf("content.SetStoreCone: %w", err)
	}
	if _, err := q.Exec("REPLACE INTO storecone(id, cone) VALUES(1, ?)", c.Serialize()); err != nil {
		return fmt.Errorf("content.SetStoreCone: %w", err)
	}
	return nil
}

// Absence classifies an artifact the store names: held, or absent for one
// of the three reasons of §5.2.
type Absence int

const (
	// Held is not absent: the store holds the content.
	Held Absence = iota
	// Transit is wanted and not yet arrived; synchronization requests it.
	Transit
	// Shunned is forbidden and never requested.
	Shunned
	// Elided is deliberately absent under the store cone. Synchronization
	// does not request it, and its absence is healthy.
	Elided
)

func (a Absence) String() string {
	switch a {
	case Held:
		return "held"
	case Transit:
		return "transit"
	case Shunned:
		return "shunned"
	case Elided:
		return "elided"
	}
	return fmt.Sprintf("Absence(%d)", int(a))
}

// Absences is the classification of every phantom in a store, computed from
// the store cone at load time. Elision is never stored on a phantom (§5.2):
// a cone change is a reload, not a migration.
type Absences struct {
	cone   *cone.Cone
	class  map[libfossil.FslID]Absence
	counts [Elided + 1]int
	work   ReferenceWork // what classification read, for tests to bound
}

// LoadAbsences classifies the store's phantoms. Shunned names win; raw F/S
// provenance proves elision only outside every covered alias and structural
// reference. Retained delta ancestors remain transit. Crosslink references
// supplement raw provenance for historical names, including prior rename paths.
// A partial tree need not be crosslinked to classify its outside S children.
// A content horizon in the cone also elides the blobs below it and the
// clusters the store was sent.
func LoadAbsences(q db.Querier) (*Absences, error) {
	return LoadAbsencesContext(context.Background(), q)
}

// LoadAbsencesContext carries cancellation through raw provenance discovery.
func LoadAbsencesContext(ctx context.Context, q db.Querier) (*Absences, error) {
	return loadAbsences(ctx, q, defaultReferenceLimits)
}

func loadAbsences(ctx context.Context, q db.Querier, limits referenceLimits) (*Absences, error) {
	if ctx == nil || q == nil {
		panic("content.LoadAbsencesContext: nil context or querier")
	}
	if err := ctx.Err(); err != nil {
		return nil, err
	}
	c, err := StoreCone(q)
	if err != nil {
		return nil, err
	}
	a := &Absences{cone: c, class: map[libfossil.FslID]Absence{}}
	hasHorizon := false
	if c != nil {
		_, hasHorizon = c.Horizon()
	}
	rows, err := q.Query(`
		SELECT p.rid,
		       EXISTS(SELECT 1 FROM shun WHERE uuid=b.uuid),
		       EXISTS(SELECT 1 FROM delta d JOIN blob h ON h.rid=d.rid
		              WHERE d.srcid=p.rid AND h.size>=0),
		       EXISTS(SELECT 1 FROM tagxref WHERE tagid=7 AND rid=p.rid)
		FROM phantom p JOIN blob b ON b.rid=p.rid`)
	if err != nil {
		return nil, fmt.Errorf("content.LoadAbsences: phantoms: %w", err)
	}
	candidates := map[libfossil.FslID]bool{}
	for rows.Next() {
		var rid libfossil.FslID
		var shunned, base, cluster bool
		if err := rows.Scan(&rid, &shunned, &base, &cluster); err != nil {
			rows.Close()
			return nil, fmt.Errorf("content.LoadAbsences: phantoms: %w", err)
		}
		switch {
		case shunned:
			a.class[rid] = Shunned
		case cluster && hasHorizon && !base:
			// A cluster the store was sent and did not keep (horizon §4.5):
			// the cluster tag outlives its content, and a horizon store
			// never asks for it back.
			a.class[rid] = Elided
		default:
			a.class[rid] = Transit
			if c != nil && !base {
				candidates[rid] = true
			}
		}
	}
	if err := errors.Join(rows.Err(), rows.Close()); err != nil {
		return nil, fmt.Errorf("content.LoadAbsences: phantoms: %w", err)
	}
	if len(candidates) > 0 {
		if err := a.classify(ctx, q, candidates, limits); err != nil {
			return nil, err
		}
	}
	for _, k := range a.class {
		a.counts[k]++
	}
	return a, nil
}

// classify decides the candidates, the phantoms that are neither shunned
// nor a held delta's base. Path members elide by name. A content horizon
// (draft-fossil-horizon-00 §3.2, §4) then elides every blob no check-in in
// its held set names, so a kept blob is in both keep sets; it never elides
// structure, and under one path members do not either.
func (a *Absences) classify(
	ctx context.Context, q db.Querier, candidates map[libfossil.FslID]bool, limits referenceLimits,
) error {
	facts, work, err := loadConeReferenceFactsBounded(ctx, q, limits.refs, true)
	a.work = work
	if err != nil {
		return err
	}
	refs, err := facts.Classify(ctx, a.cone)
	if err != nil {
		return err
	}
	if err := keepRequiredPhantoms(q, candidates, refs); err != nil {
		return err
	}
	if err := a.elide(q, candidates); err != nil {
		return err
	}
	h, hasHorizon := a.cone.Horizon()
	for rid := range candidates {
		if hasHorizon && refs.structure(rid) {
			a.class[rid] = Transit
		} else if refs.Outside(rid) {
			a.class[rid] = Elided
		} else if refs.Required(rid) {
			a.class[rid] = Transit
		}
	}
	if !hasHorizon {
		return nil
	}
	above, placed, err := horizonAbove(ctx, q, h, limits, &a.work)
	if err != nil || !placed {
		return err
	}
	for rid := range candidates {
		if refs.below(above, rid) {
			a.class[rid] = Elided
		}
	}
	return nil
}

// An artifact may be both a file's content and a structural reference or
// delta ancestor. Its out-of-cone filename does not permit either hole.
func keepRequiredPhantoms(q db.Querier, candidates map[libfossil.FslID]bool, refs *ConeReferences) error {
	var haveForum bool
	if err := q.QueryRow(`SELECT EXISTS(
		SELECT 1 FROM sqlite_master WHERE name='forumpost' AND type='table')`).
		Scan(&haveForum); err != nil {
		return fmt.Errorf("content.LoadAbsences: forum table: %w", err)
	}
	query := `
		WITH RECURSIVE retained(rid) AS (
		  SELECT d.srcid FROM delta d JOIN blob b ON b.rid=d.rid WHERE b.size>=0
		  UNION SELECT d.srcid FROM delta d JOIN retained r ON d.rid=r.rid
		)
		SELECT rid FROM retained
		UNION SELECT objid FROM event
		UNION SELECT mid FROM mlink
		UNION SELECT pid FROM plink
		UNION SELECT cid FROM plink
		UNION SELECT baseid FROM plink
		UNION SELECT rid FROM orphan
		UNION SELECT srcid FROM tagxref
		UNION SELECT origid FROM tagxref
		UNION SELECT rid FROM tagxref
		UNION SELECT attachid FROM attachment
		UNION SELECT b.rid FROM attachment a JOIN blob b ON b.uuid=a.src
		UNION SELECT parentid FROM cherrypick
		UNION SELECT childid FROM cherrypick`
	if haveForum {
		query += `
		UNION SELECT fpid FROM forumpost
		UNION SELECT froot FROM forumpost
		UNION SELECT fprev FROM forumpost
		UNION SELECT firt FROM forumpost`
	}
	rows, err := q.Query(query + " INTERSECT SELECT rid FROM phantom")
	if err != nil {
		return fmt.Errorf("content.LoadAbsences: structural references: %w", err)
	}
	for rows.Next() {
		var rid libfossil.FslID
		if err := rows.Scan(&rid); err != nil {
			rows.Close()
			return fmt.Errorf("content.LoadAbsences: structural references: %w", err)
		}
		delete(candidates, rid)
	}
	if err := errors.Join(rows.Err(), rows.Close()); err != nil {
		return fmt.Errorf("content.LoadAbsences: structural references: %w", err)
	}
	// orphan.baseline also records deferred S children, not just B baselines.
	// Only raw tree provenance can distinguish a healthy outside subtree.
	rows, err = q.Query("SELECT baseline FROM orphan")
	if err != nil {
		return err
	}
	for rows.Next() {
		var rid libfossil.FslID
		if err := rows.Scan(&rid); err != nil {
			return errors.Join(err, rows.Close())
		}
		if !refs.tree[refs.names[rid]] || !refs.Outside(rid) {
			delete(candidates, rid)
		}
	}
	return errors.Join(rows.Err(), rows.Close())
}

// elide marks the candidates F cards name only outside the cone.
func (a *Absences) elide(q db.Querier, candidates map[libfossil.FslID]bool) error {
	rows, err := q.Query(`
		SELECT m.fid, f.name FROM phantom p
		  JOIN mlink m ON m.fid=p.rid JOIN filename f ON f.fnid=m.fnid
		UNION
		SELECT m.pid, f.name FROM phantom p
		  JOIN mlink m ON m.pid=p.rid
		  JOIN filename f ON f.fnid=CASE WHEN m.pfnid>0 THEN m.pfnid ELSE m.fnid END`)
	if err != nil {
		return fmt.Errorf("content.LoadAbsences: references: %w", err)
	}
	out := map[libfossil.FslID]bool{}
	in := map[libfossil.FslID]bool{}
	for rows.Next() {
		var rid libfossil.FslID
		var name string
		if err := rows.Scan(&rid, &name); err != nil {
			rows.Close()
			return fmt.Errorf("content.LoadAbsences: references: %w", err)
		}
		if !candidates[rid] {
			continue
		}
		if a.cone.Covers(name) {
			in[rid] = true
		} else {
			out[rid] = true
		}
	}
	if err := errors.Join(rows.Err(), rows.Close()); err != nil {
		return fmt.Errorf("content.LoadAbsences: references: %w", err)
	}
	for rid := range out {
		if !in[rid] {
			a.class[rid] = Elided
		}
	}
	return nil
}

// Cone is the store cone the classification used, nil for a dense store.
func (a *Absences) Cone() *cone.Cone { return a.cone }

// Class reports why rid is absent, or Held when it is not a phantom.
func (a *Absences) Class(rid libfossil.FslID) Absence {
	if k, ok := a.class[rid]; ok {
		return k
	}
	return Held
}

// Count is the number of phantoms in class k. Held is not counted.
func (a *Absences) Count(k Absence) int {
	if k == Held {
		panic("content.Absences.Count: held artifacts are not counted")
	}
	return a.counts[k]
}