Artifact d15eda756c2592ff02dae0c93ad9551700969890ce0ab0629aad718e1d35f08a:
- File
internal/xfer/encode.go
— part of check-in
[0ca5133c62]
at
2026-07-23 10:36:32
on branch trunk
— clone: emit cfile with stored content per §7.2, not expanded file cards (#131)
* fix: clone emits cfile with stored content per sec7.2, not expanded file
Our clone server was building xfer.FileCard from cache.Expand(...) -- every artifact fully decompressed, and every delta materialized into full content -- where sec7.2 requires cfile cards carrying the repository's stored compressed content retransmitted verbatim. On real repositories this ran 80-220x more bytes on the wire than the protocol intends, and was the real blocker behind #88's clone-batching fix being unable to bound repositories much past libfossil's own size.
emitCloneBatch now reads blob.Load's stored bytes directly (full content for a whole artifact, delta bytes against DeltaSource for a deltified one) and emits xfer.CFileCard instead of xfer.FileCard, with DeltaSrc and USize (the full reconstructed size, from blob.size) populated correctly for delta rows. content.DeltaSource is exported so this and other callers outside internal/content can ask whether a rid is stored as a delta without duplicating the query.
No client changes were needed: session.go already routes CFileCard through the same handleFileCard path as FileCard, and storeDeltaContent already stores a received delta exactly as given via a create-phantom-if-missing lookup on its source, resolving delta-before-base arrival the way #53's phantom storage model was built for.
DefaultCloneBatchBytes is left at its current value rather than retuned toward canonical's 5,000,000 -- that constant's tuning is #109's territory now that bytesSent tracks stored bytes instead of expanded ones, and retuning it here risks colliding with that work.
* fix: expand deltified rows during clone instead of sending delta cfile
Code review reproduced a real interop regression: emitCloneBatch sends cards in ascending-rid order, but content.Deltify always deltifies the older artifact against the newer one, so a delta's DeltaSource almost always names a rid greater than the delta card's own rid. That means a delta cfile routinely reaches the client before the very source card it names. libfossil's own client tolerates the forward reference (storeDeltaContent phantoms the unseen source and resolves it later), but real fossil 2.28 does not -- the source stays a genuine, unfilled phantom, and content_get() on it during the client's post-clone rebuild pass returns without initializing its output blob, tripping an assertion in blob_copy (blob.c:397) and aborting the client mid-rebuild. Verified independently against a real fossil 2.28 binary cloning a libfossil server holding a delta chain.
emitCloneBatch now expands a deltified row to full content before sending it as a plain (non-delta) cfile, rather than forwarding the stored delta bytes with DeltaSrc set. The card is still zlib-compressed, so the wire-size win over the old FileCard/Expand path survives for every row; only the additional delta-retransmission savings are given up, and only for deltified rows. Re-enabling that needs either a source-before-dependent send order or confirmed forward-reference tolerance in canonical, and is left as a follow-up rather than blocking this fix.
TestEmitCloneBatchExpandsDeltifiedRows (renamed from TestEmitCloneBatchRetransmitsDeltaAsDelta) now pins the new contract: no DeltaSrc, full-sized payload. TestCloneRealFossilWithDeltaChain adds an end-to-end check against a real fossil binary, when one is on PATH. (user: [email protected], size: 9144) [annotate] [blame] [check-ins using]
package xfer import ( "bytes" "compress/zlib" "fmt" "github.com/danmestas/go-libfossil/internal/deck" ) // EncodeCard writes the wire-format representation of a Card to w. func EncodeCard(w *bytes.Buffer, c Card) error { if w == nil { panic("xfer.EncodeCard: w must not be nil") } if c == nil { panic("xfer.EncodeCard: c must not be nil") } switch v := c.(type) { case *IGotCard: return encodeIGot(w, v) case *GimmeCard: return encodeGimme(w, v) case *PushCard: return encodePush(w, v) case *PullCard: return encodePull(w, v) case *CookieCard: return encodeCookie(w, v) case *ReqConfigCard: return encodeReqConfig(w, v) case *PrivateCard: return encodePrivate(w) case *CloneCard: return encodeClone(w, v) case *CloneSeqNoCard: return encodeCloneSeqNo(w, v) case *UVGimmeCard: return encodeUVGimme(w, v) case *PragmaCard: return encodePragma(w, v) case *LoginCard: return encodeLogin(w, v) case *ErrorCard: return encodeError(w, v) case *MessageCard: return encodeMessage(w, v) case *UVIGotCard: return encodeUVIGot(w, v) case *FileCard: return encodeFile(w, v) case *CFileCard: return encodeCFile(w, v) case *ConfigCard: return encodeConfig(w, v) case *UVFileCard: return encodeUVFile(w, v) case *SchemaCard: return encodeSchema(w, v) case *XIGotCard: return encodeXIGot(w, v) case *XGimmeCard: return encodeXGimme(w, v) case *XRowCard: return encodeXRow(w, v) case *XDeleteCard: return encodeXDelete(w, v) case *UnknownCard: return encodeUnknown(w, v) default: return fmt.Errorf("xfer: cannot encode %T", c) } } func encodeIGot(w *bytes.Buffer, c *IGotCard) error { w.WriteString("igot ") w.WriteString(c.UUID) if c.IsPrivate { w.WriteString(" 1") } w.WriteByte('\n') return nil } func encodeGimme(w *bytes.Buffer, c *GimmeCard) error { w.WriteString("gimme ") w.WriteString(c.UUID) w.WriteByte('\n') return nil } // encodePush writes "push <project-code> [<server-code>]\n". // // The Fossil C xfer parser (src/xfer.c xfer_push_card) requires at least the // project code; a bare "push\n" is rejected by real Fossil servers. The // project code must always be present — callers must populate PushCard.ProjectCode // before encoding. ServerCode is optional (omitted on the first round when no // prior session exists). // // Wire-format arg order matches Fossil C: project-code first, server-code second. func encodePush(w *bytes.Buffer, c *PushCard) error { if c.ProjectCode == "" { panic("xfer: encodePush: ProjectCode must not be empty") } w.WriteString("push ") w.WriteString(c.ProjectCode) if c.ServerCode != "" { w.WriteByte(' ') w.WriteString(c.ServerCode) } w.WriteByte('\n') return nil } // encodePull writes "pull <project-code> <server-code>\n". // // The Fossil C xfer parser requires both args on pull — a bare "pull\n" or // single-arg "pull <code>\n" is rejected. Both ProjectCode and ServerCode must // be non-empty before encoding. // // Wire-format arg order matches Fossil C: project-code first, server-code second. func encodePull(w *bytes.Buffer, c *PullCard) error { if c.ProjectCode == "" { panic("xfer: encodePull: ProjectCode must not be empty") } if c.ServerCode == "" { panic("xfer: encodePull: ServerCode must not be empty") } w.WriteString("pull ") w.WriteString(c.ProjectCode) w.WriteByte(' ') w.WriteString(c.ServerCode) w.WriteByte('\n') return nil } func encodeCookie(w *bytes.Buffer, c *CookieCard) error { w.WriteString("cookie ") w.WriteString(c.Value) w.WriteByte('\n') return nil } func encodeReqConfig(w *bytes.Buffer, c *ReqConfigCard) error { w.WriteString("reqconfig ") w.WriteString(c.Name) w.WriteByte('\n') return nil } func encodePrivate(w *bytes.Buffer) error { w.WriteString("private\n") return nil } func encodeClone(w *bytes.Buffer, c *CloneCard) error { if !c.SeqNoIsDecimal && c.Version == 0 && c.SeqNo == 0 { w.WriteString("clone\n") } else { fmt.Fprintf(w, "clone %d %d\n", c.Version, c.SeqNo) } return nil } func encodeCloneSeqNo(w *bytes.Buffer, c *CloneSeqNoCard) error { fmt.Fprintf(w, "clone_seqno %d\n", c.SeqNo) return nil } func encodeUVGimme(w *bytes.Buffer, c *UVGimmeCard) error { w.WriteString("uvgimme ") w.WriteString(c.Name) w.WriteByte('\n') return nil } func encodePragma(w *bytes.Buffer, c *PragmaCard) error { w.WriteString("pragma ") w.WriteString(c.Name) for _, v := range c.Values { w.WriteByte(' ') w.WriteString(v) } w.WriteByte('\n') return nil } func encodeLogin(w *bytes.Buffer, c *LoginCard) error { w.WriteString("login ") w.WriteString(deck.FossilEncode(c.User)) w.WriteByte(' ') w.WriteString(c.Nonce) w.WriteByte(' ') w.WriteString(c.Signature) w.WriteByte('\n') return nil } func encodeError(w *bytes.Buffer, c *ErrorCard) error { w.WriteString("error ") w.WriteString(deck.FossilEncode(c.Message)) w.WriteByte('\n') return nil } func encodeMessage(w *bytes.Buffer, c *MessageCard) error { w.WriteString("message ") w.WriteString(deck.FossilEncode(c.Message)) w.WriteByte('\n') return nil } func encodeUVIGot(w *bytes.Buffer, c *UVIGotCard) error { fmt.Fprintf(w, "uvigot %s %d %s %d\n", c.Name, c.MTime, c.Hash, c.Size) return nil } func encodeUnknown(w *bytes.Buffer, c *UnknownCard) error { w.WriteString(c.Command) for _, a := range c.Args { w.WriteByte(' ') w.WriteString(a) } w.WriteByte('\n') return nil } // --- Payload card encoders --- // encodeFile writes: file UUID SIZE \n CONTENT (no trailing \n) // or: file UUID DELTASRC SIZE \n CONTENT (delta variant) func encodeFile(w *bytes.Buffer, c *FileCard) error { w.WriteString("file ") w.WriteString(c.UUID) if c.DeltaSrc != "" { w.WriteByte(' ') w.WriteString(c.DeltaSrc) } fmt.Fprintf(w, " %d\n", len(c.Content)) w.Write(c.Content) // NO trailing newline return nil } // encodeCFile writes: cfile UUID USIZE CSIZE \n ZCONTENT (no trailing \n) // or: cfile UUID DELTASRC USIZE CSIZE \n ZCONTENT (delta variant) // // §7.2: USIZE is the sender's full reconstructed artifact size, which for a // delta card is not len(Content) -- Content there is the (smaller) delta // payload. A caller that knows the reconstructed size (the clone send path // does, from blob.size) sets c.USize; callers that don't -- every non-delta // caller, where the two quantities coincide -- leave it zero and get the old // len(Content) behaviour. func encodeCFile(w *bytes.Buffer, c *CFileCard) error { // Compress content with zlib var zbuf bytes.Buffer zw := zlib.NewWriter(&zbuf) if _, err := zw.Write(c.Content); err != nil { return fmt.Errorf("xfer: cfile zlib write: %w", err) } if err := zw.Close(); err != nil { return fmt.Errorf("xfer: cfile zlib close: %w", err) } usize := c.USize if usize == 0 { usize = len(c.Content) } w.WriteString("cfile ") w.WriteString(c.UUID) if c.DeltaSrc != "" { w.WriteByte(' ') w.WriteString(c.DeltaSrc) } fmt.Fprintf(w, " %d %d\n", usize, zbuf.Len()) w.Write(zbuf.Bytes()) // NO trailing newline return nil } // encodeConfig writes: config NAME SIZE \n CONTENT \n // Note the trailing \n after CONTENT. func encodeConfig(w *bytes.Buffer, c *ConfigCard) error { fmt.Fprintf(w, "config %s %d\n", c.Name, len(c.Content)) w.Write(c.Content) w.WriteByte('\n') // trailing newline return nil } // encodeUVFile writes: uvfile NAME MTIME HASH SIZE FLAGS \n CONTENT // When deleted (0x0001) or content-omitted (0x0004), no payload follows. func encodeUVFile(w *bytes.Buffer, c *UVFileCard) error { fmt.Fprintf(w, "uvfile %s %d %s %d %d\n", c.Name, c.MTime, c.Hash, c.Size, c.Flags) if !uvFileOmitsContent(c.Flags) { w.Write(c.Content) } // NO trailing newline return nil } func encodeSchema(w *bytes.Buffer, c *SchemaCard) error { fmt.Fprintf(w, "schema %s %d %s %d %d\n", c.Table, c.Version, c.Hash, c.MTime, len(c.Content)) w.Write(c.Content) w.WriteByte('\n') return nil } func encodeXIGot(w *bytes.Buffer, c *XIGotCard) error { if c.Table == "" { panic("encodeXIGot: Table must not be empty") } if c.PKHash == "" { panic("encodeXIGot: PKHash must not be empty") } fmt.Fprintf(w, "xigot %s %s %d\n", c.Table, c.PKHash, c.MTime) return nil } func encodeXGimme(w *bytes.Buffer, c *XGimmeCard) error { if c.Table == "" { panic("encodeXGimme: Table must not be empty") } if c.PKHash == "" { panic("encodeXGimme: PKHash must not be empty") } fmt.Fprintf(w, "xgimme %s %s\n", c.Table, c.PKHash) return nil } func encodeXRow(w *bytes.Buffer, c *XRowCard) error { if c.Table == "" { panic("encodeXRow: Table must not be empty") } if c.PKHash == "" { panic("encodeXRow: PKHash must not be empty") } fmt.Fprintf(w, "xrow %s %s %d %d\n", c.Table, c.PKHash, c.MTime, len(c.Content)) w.Write(c.Content) w.WriteByte('\n') return nil } func encodeXDelete(w *bytes.Buffer, c *XDeleteCard) error { if c.Table == "" { panic("encodeXDelete: Table must not be empty") } if c.PKHash == "" { panic("encodeXDelete: PKHash must not be empty") } fmt.Fprintf(w, "xdelete %s %s %d %d\n", c.Table, c.PKHash, c.MTime, len(c.PKData)) w.Write(c.PKData) w.WriteByte('\n') return nil }