Artifact [b161701e17]

Artifact b161701e17c57c7f064c84ca122557e435f18fccf1a9bd8e564b6bd37b1e27e2:

• File internal/content/stored_chunk_test.go — part of check-in [1fd12b3ed3] at 2026-10-28 21:53:09 on branch feat/chunked-blobs — Store blobs over a hosted value limit across blobpart rows

  A hosted SQLite that caps value size (a Durable Object's 2 MB) cannot hold a large file's stored blob in one blob.content value. With the new db.DriverConfig.MaxValueBytes set, a stored value over the limit is written in 1,000,000-byte parts to blobpart(rid, seq, content), created on first use, leaving blob.content NULL and blob.size the expanded length, so phantoms (size < 0) stay unambiguous. Chunking sits below compression: hashes, deltas and the wire format are unchanged.

  internal/blob owns the seam (ReadStored, WriteStored, InsertStored, StoredHead, DropStored, SweepParts and two scan expressions), and every direct reader or writer of blob.content now goes through it: delta rewrite and undelta, receive, clone send and walk, compaction, sparse demotion, seed drop, shun purge, horizon cost and cone preparation.

  Zero, the default, keeps every value inline and creates no table, so local stores stay byte-identical. A store with blobpart rows is not readable by stock fossil; enable it only for hosted backends.

  Closes #459 (user: [email protected], size: 3403) [annotate] [blame] [check-ins using]

package content

import (
	"bytes"
	"context"
	"math/rand"
	"testing"
	"time"

	"github.com/danmestas/go-libfossil/db"
	"github.com/danmestas/go-libfossil/internal/blob"
	"github.com/danmestas/go-libfossil/internal/deck"
	libfossil "github.com/danmestas/go-libfossil/internal/fsltype"
)

func chunkBytes(n int, seed int64) []byte {
	b := make([]byte, n)
	_, _ = rand.New(rand.NewSource(seed)).Read(b)
	return b
}

// partsOf counts rid's blobpart rows.
func partsOf(t *testing.T, q db.Querier, rid libfossil.FslID) int {
	t.Helper()
	var n int
	if err := q.QueryRow("SELECT count(*) FROM blobpart WHERE rid=?", rid).Scan(&n); err != nil {
		t.Fatal(err)
	}
	return n
}

func expandEquals(t *testing.T, q db.Querier, rid libfossil.FslID, want []byte) {
	t.Helper()
	got, err := Expand(q, rid)
	if err != nil {
		t.Fatalf("Expand rid=%d: %v", rid, err)
	}
	if !bytes.Equal(got, want) {
		t.Fatalf("Expand rid=%d: content differs", rid)
	}
}

// Deltifying a chunked row rewrites it inline and drops its parts; undoing
// that chunks it again.
func TestDeltifyAndUndeltaChunkedRow(t *testing.T) {
	t.Cleanup(blob.SetMaxValueBytes(1_990_000))
	d := setupTestDB(t)
	old := chunkBytes(2_100_000, 1)
	cur := bytes.Clone(old)
	copy(cur[500_000:], "an edit")
	oldRid, _, err := blob.Store(d, old)
	if err != nil {
		t.Fatal(err)
	}
	curRid, _, err := blob.Store(d, cur)
	if err != nil {
		t.Fatal(err)
	}
	if partsOf(t, d, oldRid) != 3 {
		t.Fatalf("fixture: rid %d is not chunked", oldRid)
	}
	err = d.WithTx(func(tx *db.Tx) error {
		saved, err := Deltify(tx, oldRid, curRid)
		if err == nil && saved <= 2_000_000 {
			t.Errorf("Deltify saved %d bytes, want the chunked value's worth", saved)
		}
		return err
	})
	if err != nil {
		t.Fatal(err)
	}
	if partsOf(t, d, oldRid) != 0 {
		t.Fatal("Deltify kept the rewritten row's parts")
	}
	expandEquals(t, d, oldRid, old)

	if err := d.WithTx(func(tx *db.Tx) error { return Undelta(tx, oldRid) }); err != nil {
		t.Fatal(err)
	}
	if partsOf(t, d, oldRid) != 3 {
		t.Fatal("Undelta did not chunk the full value")
	}
	expandEquals(t, d, oldRid, old)
	expandEquals(t, d, curRid, cur)
}

// Compaction turns a chunked row outside the cone into a phantom and takes
// its parts with it.
func TestCompactStoreDropsParts(t *testing.T) {
	t.Cleanup(blob.SetMaxValueBytes(1_990_000))
	d := setupTestDB(t)
	out, outName, err := blob.Store(d, chunkBytes(2_100_000, 2))
	if err != nil {
		t.Fatal(err)
	}
	inside := chunkBytes(2_100_000, 3)
	in, inName, err := blob.Store(d, inside)
	if err != nil {
		t.Fatal(err)
	}
	manifest, err := (&deck.Deck{Type: deck.Checkin, D: time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC),
		F: []deck.FileCard{{Name: "docs/a", UUID: outName}, {Name: "src/a", UUID: inName}},
		U: deck.User("test")}).Marshal()
	if err != nil {
		t.Fatal(err)
	}
	if _, _, err := blob.Store(d, manifest); err != nil {
		t.Fatal(err)
	}
	if err := SetStoreCone(d, mustCone(t, []string{"src"}, nil)); err != nil {
		t.Fatal(err)
	}
	if n, err := CompactStore(context.Background(), d); err != nil || n != 1 {
		t.Fatalf("CompactStore = %d, %v; want 1, nil", n, err)
	}
	if partsOf(t, d, out) != 0 {
		t.Fatal("compaction kept the dropped row's parts")
	}
	if _, err := blob.ReadStored(d, out); err == nil {
		t.Fatal("compacted row still reads")
	}
	if partsOf(t, d, in) != 3 {
		t.Fatal("compaction touched the kept row's parts")
	}
	expandEquals(t, d, in, inside)
}