package index
import (
"crypto/md5"
"encoding/hex"
"os"
"os/exec"
"path/filepath"
"strconv"
"strings"
"testing"
"craftdesign.group/quarry/internal/artifact"
)
// This file is the regression net for quarry's worst failure mode. An
// artifact that fails to parse is not an error anybody sees: ingest logs
// a line and stores it as opaque content, so the record simply is not
// there. On 2026-08-16 that cost sqlite every one of its 794 tickets and
// 1,009 check-ins across two corpora, and what found it was a person
// noticing the forum page was empty.
//
// Each case below is a shape that real fossil repositories contain and
// that quarry once refused. The assertion is deliberately end-to-end for
// the silent part — parse, then fold, then look for the row — because
// parsing correctly and folding into nothing loses the record just as
// completely.
//
// Ground truth for the counts lives in fossil's own tables; see
// TestCorpusParity for the check against a real corpus.
// seal appends the Z self-checksum an artifact needs to pass §6.2.
func seal(body string) []byte {
sum := md5.Sum([]byte(body))
return []byte(body + "Z " + hex.EncodeToString(sum[:]) + "\n")
}
const (
h40 = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
h64 = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
)
func TestAwkwardArtifactsSurviveIngest(t *testing.T) {
cases := []struct {
name string
body string
// check runs after the artifact is folded; it must find the
// record, not merely observe that parsing did not error.
check func(*testing.T, *Index)
}{
{
// The single largest class of loss: fossil writes an empty
// comment as "C" SP LF, and a reader that rejects it drops
// the whole check-in, not just the comment text.
name: "check-in with an empty comment",
body: "C \nD 2026-08-15T10:00:00\nU dan\n",
check: func(t *testing.T, ix *Index) {
tl, _, err := ix.Timeline(TimelineFilter{Limit: 10, Kind: "ci"})
if err != nil {
t.Fatal(err)
}
if len(tl) != 1 {
t.Fatalf("empty-comment check-in did not land: %+v", tl)
}
if tl[0].Comment != "" {
t.Errorf("comment = %q, want empty", tl[0].Comment)
}
},
},
{
// sqlite's tickets are full of these: a field present with an
// empty value, written as a bare trailing space.
name: "ticket change with an empty field value",
body: "D 2026-08-15T10:00:00\nJ comment \nJ status Open\nK " + h40 + "\nU dan\n",
check: func(t *testing.T, ix *Index) {
st, ok, err := ix.TicketByPrefix(h40[:10])
if err != nil {
t.Fatal(err)
}
if !ok {
t.Fatal("ticket with an empty field value did not land")
}
if st.ID != h40 {
t.Errorf("ticket id = %q", st.ID)
}
},
},
{
// Tag values are optional AND may be present-but-empty; the
// two are different (absent is null, empty is the empty
// string) and both occur in tcl.
name: "tag with an empty value",
body: "D 2026-08-15T10:00:00\nT +date " + h64 + " \nU dan\n",
check: func(t *testing.T, ix *Index) {
// A control artifact folds tags, not a timeline row; the
// check is that it parsed as structural at all.
},
},
{
name: "technote carrying a title",
body: "C Release\\snotes\nD 2026-08-15T10:00:00\nE 2026-08-15T09:00:00 " + h40 +
"\nN text/x-markdown\nU dan\nW 6\nhello\n\n",
check: func(t *testing.T, ix *Index) {
notes, err := ix.Technotes()
if err != nil {
t.Fatal(err)
}
if len(notes) != 1 {
t.Fatalf("technote did not land: %+v", notes)
}
if notes[0].Comment != "Release notes" {
t.Errorf("technote title = %q", notes[0].Comment)
}
},
},
{
// An omitted N card is text/x-fossil-wiki by fossil's rule,
// NOT markdown. Folding it as empty is what lets the render
// side apply that dispatch.
name: "forum post without a mimetype card",
body: "D 2026-08-15T10:00:00\nG " + h64 + "\nH A\\sthread\nU dan\nW 5\nbody\n\n",
check: func(t *testing.T, ix *Index) {
threads, err := ix.ForumThreads(0, "")
if err != nil {
t.Fatal(err)
}
if len(threads) != 1 {
t.Fatalf("forum post did not land: %+v", threads)
}
},
},
{
name: "wiki page",
body: "D 2026-08-15T10:00:00\nL Home\nU dan\nW 6\nhello\n\n",
check: func(t *testing.T, ix *Index) {
titles, err := ix.WikiTitles()
if err != nil {
t.Fatal(err)
}
if len(titles) != 1 || titles[0].Title != "Home" {
t.Fatalf("wiki page did not land: %+v", titles)
}
},
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
ix := openIndex(t)
raw := seal(tc.body)
art, err := artifact.Parse(raw)
if err != nil {
t.Fatalf("rejected an artifact real repositories contain: %v", err)
}
if art == nil {
t.Fatal("classified as opaque content — this is the silent loss: " +
"ingest would log a line and the record would vanish")
}
hash := strings.Repeat("c", 64)
if err := ix.FoldNow([]Parsed{{Hash: hash, Art: art}}); err != nil {
t.Fatalf("fold: %v", err)
}
tc.check(t, ix)
})
}
}
// Content must never come back as an artifact. The contract is subtler
// than "(nil, nil)": a blob whose first line is card-SHAPED gets probed
// as structural and fails at the card level, so Parse returns (nil,
// error) and ingest treats a nil artifact as content whatever the error
// says. The essential property is that no bogus artifact is produced.
//
// The error on a plain file is also why the ingest log reads like
// corruption for tcl's "A NOTE ON THE CHANGELOG:" — see the ticket on
// separating a content probe from a real parse failure.
func TestContentStaysContent(t *testing.T) {
for _, blob := range []string{
"A NOTE ON THE CHANGELOG:\nStarting in early 2011...\n", // card-shaped, probed, fails
"plain text\n",
"",
} {
art, err := artifact.Parse([]byte(blob))
if art != nil {
t.Errorf("%.20q: parsed as an artifact (%v), want content", blob, art)
}
_ = err // may be set: a failed structural probe is still content
}
}
// TestCorpusParity compares a built index against fossil's own tables on
// a real repository — the check that actually found both 2026-08-16
// data-loss bugs, made repeatable.
//
// It is opt-in because the corpora are hundreds of megabytes and fossil
// must be installed: set QUARRY_PARITY to a directory holding
// <name>.fossil clones, and QUARRY_PARITY_INDEX to a quarry data
// directory holding <name>/index.db. Skipped otherwise, so CI stays
// self-contained while a maintainer can run the real thing:
//
// QUARRY_PARITY=~/quarry-corpus/fossil \
// QUARRY_PARITY_INDEX=~/quarry-demo/data \
// go test ./internal/index/ -run CorpusParity -v
func TestCorpusParity(t *testing.T) {
corpus, indexes := os.Getenv("QUARRY_PARITY"), os.Getenv("QUARRY_PARITY_INDEX")
if corpus == "" || indexes == "" {
t.Skip("set QUARRY_PARITY and QUARRY_PARITY_INDEX to compare against fossil")
}
if _, err := exec.LookPath("sqlite3"); err != nil {
t.Skip("sqlite3 not on PATH")
}
names, err := filepath.Glob(filepath.Join(corpus, "*.fossil"))
if err != nil || len(names) == 0 {
t.Skipf("no .fossil clones under %s", corpus)
}
// Each row is one count that must agree. The fossil query is ground
// truth; the quarry query reads the derived index. Attachments are
// included because an A card is an event on its target in fossil,
// which is what made ticket-change counts disagree before they were
// folded at all.
checks := []struct {
what string
fossil string
quarry string
}{
{"check-ins", `select count(*) from event where type='ci'`, `select count(*) from checkin`},
{"tickets", `select count(*) from ticket`, `select count(distinct ticket) from ticket_change`},
// A CENSUS comparison, deliberately: both sides count every page
// that has ever existed, deleted included, because this check is
// about ingest completeness. The LIVE list (WikiTitles, Counts)
// filters to pages whose newest edit is non-empty — fossil's
// deletion convention — and agreeing here says nothing about
// that rule. #7a649ff845 is where the two were conflated.
{"wiki pages", `select count(distinct substr(tagname,6)) from tag where tagname like 'wiki-%'`,
`select count(distinct title) from wiki_edit`},
{"forum posts", `select count(*) from forumpost`, `select count(*) from forum_post`},
{"attachments", `select count(*) from attachment`, `select count(*) from attachment`},
}
// Known, understood differences. Anything NOT listed here is a
// failure — a blanket tolerance would hide the regressions this test
// exists to catch.
//
// tcl check-ins: artifact 6c333137… is a well-formed check-in
// manifest that fossil stored AND crosslinked (it has plink, mlink
// and tagxref rows) but for which it created no timeline event.
// quarry indexes it, so quarry reports one more. Investigated
// 2026-08-16; quarry's count is the defensible one.
known := map[string]int{
"tcl/check-ins": 1,
// libfossil wiki pages: fossil names a page with a wiki-<name>
// tag, and libfossil carries a tag literally called
// wiki-tag-propagation-test — applied TO the FIXMEs wiki
// artifacts as a propagation test, with no artifact whose L card
// names that page. So the tag count says 17 where 16 pages
// exist. quarry is right; the ground-truth query cannot tell a
// page from a tag that looks like one.
"libfossil/wiki pages": -1,
}
for _, path := range names {
name := strings.TrimSuffix(filepath.Base(path), ".fossil")
db := filepath.Join(indexes, name, "index.db")
if _, err := os.Stat(db); err != nil {
t.Logf("%s: no index at %s, skipping", name, db)
continue
}
t.Run(name, func(t *testing.T) {
for _, c := range checks {
want := sqliteScalar(t, path, c.fossil)
got := sqliteScalar(t, db, c.quarry)
if want == "" || got == "" {
t.Logf("%s: unavailable (fossil=%q quarry=%q)", c.what, want, got)
continue
}
if want == got {
continue
}
wn, gn := atoiOr(want, -1), atoiOr(got, -2)
if delta, ok := known[name+"/"+c.what]; ok && gn-wn == delta {
t.Logf("%s: quarry has %s, fossil has %s (known delta %+d)", c.what, got, want, delta)
continue
}
t.Errorf("%s: quarry has %s, fossil has %s", c.what, got, want)
}
})
}
}
func atoiOr(s string, fallback int) int {
n, err := strconv.Atoi(s)
if err != nil {
return fallback
}
return n
}
func sqliteScalar(t *testing.T, db, query string) string {
t.Helper()
out, err := exec.Command("sqlite3", db, query).Output()
if err != nil {
return ""
}
return strings.TrimSpace(string(out))
}