Artifact 59fec8666c47c8a4c1bf629b1a76e3ae6a3dec27ed3043729b83a08a9b60c933:
- File
internal/blob/blob.go
— part of check-in
[670ca0e75f]
at
2026-07-19 16:00:09
on branch trunk
— fix: separate blob existence from blob availability (#43) (#52)
* fix: separate blob existence from blob availability (#43)
A phantom is a real blob row with size = -1 and NULL content, so blob.Exists — "SELECT rid FROM blob WHERE uuid=?" — answers true for it. Every guard built on Exists therefore admitted artifacts whose content cannot be read, and content.Expand aborted with "rid N is a phantom".
Canonical Fossil keeps two predicates and never substitutes one for the other: rid_from_uuid (existence, xfer.c:70) and content_is_available (availability, content.c:163). The latter is transitive — it follows delta.srcid up the chain, so a delta whose base is a phantom is unavailable even though every row in the chain exists.
Add content.IsAvailable / content.AvailableByUUID, ported from content_is_available including the chain walk, with an explicit depth bound so a cyclic delta table terminates instead of hanging. The walk lives in exactly one place; no caller touches the delta table.
blob.Exists keeps its behavior and gains a doc comment stating that it answers existence only, returns true for phantoms, and naming the availability predicate as the alternative.
manifest.blobPresent is deleted. It was a local patch at one call site that tested size >= 0 without the chain walk, so it still reported a delta-on-phantom as present — the same class of bug it was written to work around.
Every blob.Exists call site was classified as existence or availability and converted where it guards a content read. Sites that want rid identity — plink/mlink/tagxref population, cluster phantom creation, private-status bookkeeping, the post-store sanity check in handleFile, and phantom filling in storeReceivedFile — keep existence, since delta-before-base is a normal steady state during clone.
Verified against a loopback fossil 2.28 server: the 23-artifact repro that previously failed roughly half the time now clones 6 of 6.
* fix: IsAvailable must fail closed when the delta lookup errors
Only sql.ErrNoRows means the chain is grounded. Any other failure — cancelled context, closed connection, NULL srcid — says nothing about groundedness, so returning true there claims readable content that was never verified. That is the same class of bug this predicate exists to prevent, and it runs on data arriving over the wire during clone.
The blob-size lookup above already fails closed on the same pattern; this restores the symmetry. (user: [email protected], size: 7433) [annotate] [blame] [check-ins using]
package blob import ( "fmt" libfossil "github.com/danmestas/libfossil/internal/fsltype" "github.com/danmestas/libfossil/db" "github.com/danmestas/libfossil/internal/delta" "github.com/danmestas/libfossil/internal/hash" ) func Store(q db.Querier, content []byte) (rid libfossil.FslID, uuid string, err error) { if q == nil { panic("blob.Store: q must not be nil") } if len(content) == 0 { panic("blob.Store: content length must be > 0") } defer func() { if err == nil && rid <= 0 { panic("blob.Store: postcondition violated: rid <= 0 with no error") } }() uuid = hash.SHA1(content) if rid, ok := Exists(q, uuid); ok { return rid, uuid, nil } compressed, err := Compress(content) if err != nil { return 0, "", fmt.Errorf("blob.Store compress: %w", err) } result, err := q.Exec( "INSERT INTO blob(uuid, size, content, rcvid) VALUES(?, ?, ?, 1)", uuid, len(content), compressed, ) if err != nil { return 0, "", fmt.Errorf("blob.Store insert: %w", err) } ridInt, err := result.LastInsertId() if err != nil { return 0, "", fmt.Errorf("blob.Store lastid: %w", err) } rid = libfossil.FslID(ridInt) // Verify round-trip: re-read, decompress, re-hash. // Matches Fossil's content_put_pk() post-write verification. readBack, err := Load(q, rid) if err != nil { return 0, "", fmt.Errorf("blob.Store verify read-back: %w", err) } if got := hash.SHA1(readBack); got != uuid { return 0, "", fmt.Errorf("blob.Store verify: hash mismatch after round-trip: stored %s, got %s", uuid, got) } // Mark as unclustered — matches Fossil's content_put_ex (content.c:633). // Only new blobs reach here; Exists early-return skips this. if _, err := q.Exec("INSERT OR IGNORE INTO unclustered(rid) VALUES(?)", rid); err != nil { return 0, "", fmt.Errorf("blob.Store unclustered: %w", err) } return rid, uuid, nil } func StoreDelta(q db.Querier, content []byte, srcRid libfossil.FslID) (rid libfossil.FslID, uuid string, err error) { if q == nil { panic("blob.StoreDelta: q must not be nil") } if len(content) == 0 { panic("blob.StoreDelta: content length must be > 0") } if srcRid <= 0 { panic("blob.StoreDelta: srcRid must be > 0") } defer func() { if err == nil && rid <= 0 { panic("blob.StoreDelta: postcondition violated: rid <= 0 with no error") } }() uuid = hash.SHA1(content) if rid, ok := Exists(q, uuid); ok { return rid, uuid, nil } srcContent, err := Load(q, srcRid) if err != nil { return 0, "", fmt.Errorf("blob.StoreDelta load source: %w", err) } deltaBytes := delta.Create(srcContent, content) compressed, err := Compress(deltaBytes) if err != nil { return 0, "", fmt.Errorf("blob.StoreDelta compress: %w", err) } result, err := q.Exec( "INSERT INTO blob(uuid, size, content, rcvid) VALUES(?, ?, ?, 1)", uuid, len(content), compressed, ) if err != nil { return 0, "", fmt.Errorf("blob.StoreDelta insert blob: %w", err) } ridInt, err := result.LastInsertId() if err != nil { return 0, "", fmt.Errorf("blob.StoreDelta lastid: %w", err) } rid = libfossil.FslID(ridInt) _, err = q.Exec("INSERT INTO delta(rid, srcid) VALUES(?, ?)", rid, srcRid) if err != nil { return 0, "", fmt.Errorf("blob.StoreDelta insert delta: %w", err) } // Verify round-trip: re-read delta, apply to source, re-hash. // Matches Fossil's content_put_pk() post-write verification. storedDelta, err := Load(q, rid) if err != nil { return 0, "", fmt.Errorf("blob.StoreDelta verify read-back: %w", err) } rebuilt, err := delta.Apply(srcContent, storedDelta) if err != nil { return 0, "", fmt.Errorf("blob.StoreDelta verify apply: %w", err) } if got := hash.SHA1(rebuilt); got != uuid { return 0, "", fmt.Errorf("blob.StoreDelta verify: hash mismatch after round-trip: stored %s, got %s", uuid, got) } // Mark as unclustered — matches Fossil's content_put_ex (content.c:633). if _, err := q.Exec("INSERT OR IGNORE INTO unclustered(rid) VALUES(?)", rid); err != nil { return 0, "", fmt.Errorf("blob.StoreDelta unclustered: %w", err) } return rid, uuid, nil } func StorePhantom(q db.Querier, uuid string) (rid libfossil.FslID, err error) { if q == nil { panic("blob.StorePhantom: q must not be nil") } if uuid == "" { panic("blob.StorePhantom: uuid must not be empty") } defer func() { if err == nil && rid <= 0 { panic("blob.StorePhantom: postcondition violated: rid <= 0 with no error") } }() if rid, ok := Exists(q, uuid); ok { return rid, nil } result, err := q.Exec( "INSERT INTO blob(uuid, size, content, rcvid) VALUES(?, -1, NULL, 0)", uuid, ) if err != nil { return 0, fmt.Errorf("blob.StorePhantom: %w", err) } ridInt, err := result.LastInsertId() if err != nil { return 0, fmt.Errorf("blob.StorePhantom lastid: %w", err) } rid = libfossil.FslID(ridInt) _, err = q.Exec("INSERT INTO phantom(rid) VALUES(?)", rid) if err != nil { return 0, fmt.Errorf("blob.StorePhantom phantom table: %w", err) } return rid, nil } func Load(q db.Querier, rid libfossil.FslID) (result []byte, err error) { if q == nil { panic("blob.Load: q must not be nil") } if rid <= 0 { panic("blob.Load: rid must be > 0") } defer func() { if err == nil && result == nil { panic("blob.Load: postcondition violated: result is nil with no error") } }() var content []byte var size int64 err = q.QueryRow("SELECT content, size FROM blob WHERE rid=?", rid).Scan(&content, &size) if err != nil { return nil, fmt.Errorf("blob.Load query: %w", err) } if size == -1 { return nil, fmt.Errorf("blob.Load: rid %d is a phantom", rid) } if content == nil || len(content) == 0 { return nil, fmt.Errorf("blob.Load: rid %d has NULL or empty content", rid) } // Fossil stores blobs as [4-byte BE uncompressed-size][zlib data]. // When the compressed form happens to be the same length as the // uncompressed content (rare but real — ~2 in 66K in the Fossil SCM // repo), we must still decompress. Detect compressed content by // checking for the 4-byte BE prefix matching the declared size // followed by a zlib header (0x78). if len(content) >= 6 { prefixSize := int64(content[0])<<24 | int64(content[1])<<16 | int64(content[2])<<8 | int64(content[3]) if prefixSize == size && content[4] == 0x78 { return Decompress(content) } } // No compression prefix — content is stored uncompressed. if int64(len(content)) == size { return content, nil } // Stored bytes < declared size — compressed. return Decompress(content) } // Exists reports whether a blob row with the given uuid is present, and // returns its rid. It answers existence only. // // It returns true for a phantom — a real blob row with size = -1 and NULL // content, standing in for an artifact we know of but have not received. // A phantom's content cannot be read, and neither can that of a delta whose // chain bottoms out in one. Callers that are about to read content must use // content.AvailableByUUID instead, which is transitive over the delta chain. // // Mirrors Fossil's rid_from_uuid (src/xfer.c:70). func Exists(q db.Querier, uuid string) (libfossil.FslID, bool) { if q == nil { panic("blob.Exists: q must not be nil") } if uuid == "" { panic("blob.Exists: uuid must not be empty") } var rid int64 err := q.QueryRow("SELECT rid FROM blob WHERE uuid=?", uuid).Scan(&rid) if err != nil { return 0, false } return libfossil.FslID(rid), true }